From aa59621c286b42d8e62bcc65200292c64ed782fd Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Mon, 2 Nov 2026 21:23:30 -0700 Subject: [PATCH 01/26] Expose opt-in full-text operation analytics --- .github/workflows/unit-test.yml | 4 +- DESIGN.md | 3 +- benchmarks/fulltext-analytics.bench.js | 279 +++++++++++ benchmarks/fulltext-analytics.mjs | 130 ++++++ config-root.schema.json | 22 + config/RootConfigWatcher.ts | 30 +- config/fullTextTelemetryConfiguration.ts | 68 +++ dependencies.md | 10 +- package-lock.json | 128 ++++- package.json | 3 +- renovate.json | 4 +- resources/FULL_TEXT.md | 36 +- resources/analytics/DESIGN.md | 8 + resources/analytics/fullTextTelemetry.ts | 442 ++++++++++++++++++ .../analytics/fullTextTelemetryRuntime.ts | 90 ++++ resources/analytics/metadata.ts | 1 + resources/analytics/write.ts | 25 +- resources/derivedIndexes.ts | 2 + resources/indexes/DESIGN.md | 2 +- resources/indexes/fullTextDerivedIndex.ts | 9 +- resources/indexes/fullTextNativeBinding.ts | 33 +- resources/indexes/fullTextQueryIndex.ts | 9 +- .../nativeFullTextDerivedIndexLifecycle.ts | 6 +- .../fullTextTelemetryConfiguration.test.js | 130 ++++++ .../analytics/aggregationCycle.test.js | 48 ++ .../derivedIndexRuntimeRocks.test.js | 11 +- .../fixtures/fullTextUpgradeProcess.cjs | 12 +- .../resources/fullTextNativeTestBinding.js | 11 +- ...ativeFullTextDerivedIndexLifecycle.test.js | 122 +++-- .../resources/nativeFullTextTelemetry.test.js | 407 ++++++++++++++++ utility/environment/environmentManager.ts | 43 ++ utility/hdbTerms.ts | 5 + validation/configValidator.ts | 6 + 33 files changed, 2045 insertions(+), 94 deletions(-) create mode 100644 benchmarks/fulltext-analytics.bench.js create mode 100644 benchmarks/fulltext-analytics.mjs create mode 100644 config/fullTextTelemetryConfiguration.ts create mode 100644 resources/analytics/fullTextTelemetry.ts create mode 100644 resources/analytics/fullTextTelemetryRuntime.ts create mode 100644 unitTests/config/fullTextTelemetryConfiguration.test.js create mode 100644 unitTests/resources/nativeFullTextTelemetry.test.js diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 8f0333e1e1..2f5a479beb 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -326,12 +326,12 @@ jobs: run: npm run build - name: Verify the published native binding loads - run: node --input-type=module --eval "const binding = await import('@harperfast/fulltext/native'); const info = await binding.runtimeInfo(); if (typeof binding.openNativeFullTextIndex !== 'function' || typeof binding.openNativeFullTextReader !== 'function' || typeof binding.NativeFullTextIndex?.prototype.searchPage !== 'function' || info.nativeAbiVersion !== 13 || info.queryApiVersion !== 7) throw new Error('@harperfast/fulltext loaded without the required writer, reader, and native ABI 13 and query API 7 contract');" + run: node --input-type=module --eval "const { validateFullTextNativeBinding } = await import('./dist/resources/indexes/fullTextNativeBinding.js'); await validateFullTextNativeBinding(await import('@harperfast/fulltext/native'));" - name: Native full-text lifecycle test run: >- npx mocha - unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js + unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js unitTests/resources/nativeFullTextTelemetry.test.js unitTests/resources/nativeFullTextTableSearch.test.js unitTests/resources/nativeFullTextIndexSchemaUpgrade.test.js unitTests/resources/nativeFullTextIndexUpgradeWorkers.test.js diff --git a/DESIGN.md b/DESIGN.md index 22d5f97dfa..d75b6edcb6 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -72,12 +72,13 @@ A note lives in the `DESIGN.md` of the directory that owns the code it describes - [A filtered vector search may admit from an index key set only if the set holds every predicate match](resources/indexes/DESIGN.md#a-filtered-vector-search-may-admit-from-an-index-key-set-only-if-the-set-holds-every-predicate-match) — An indexed companion condition becomes an HNSW allow-set only when its record predicate rejects every value the index cannot hold. - [Filtered full-text search reuses companion secondary indexes before loading source records](resources/indexes/DESIGN.md#filtered-full-text-search-reuses-companion-secondary-indexes-before-loading-source-records) — Selective ranges become bounded key sets; broad indexed equalities become costed point probes before Harper loads authoritative records. - [Derived-index runtime: committed-log delivery to native index backends (`resources/derivedIndexRuntime.ts`)](resources/indexes/DESIGN.md#derived-index-runtime-committed-log-delivery-to-native-index-backends-resourcesderivedindexruntimets) — The transaction log is the durable fact; a commit only wakes the lock-elected runner, which resumes every backend from an exact cursor. -- [Native full-text backend](resources/indexes/DESIGN.md#derived-index-runtime-committed-log-delivery-to-native-index-backends-resourcesderivedindexruntimets) — The native full-text subsection documents Tantivy files as node-local derived state, with one elected writer advancing each index while per-worker readers use publication and reset fencing. +- [Native full-text backend](resources/indexes/DESIGN.md#derived-index-runtime-committed-log-delivery-to-native-index-backends-resourcesderivedindexruntimets) — One elected writer; node-local Tantivy state and fenced per-worker readers. - [Native HNSW plane: a file-primary mmap graph on the derived-index runtime (`resources/indexes/HierarchicalNavigableSmallWorld.ts`, `resources/indexes/hnswDerivedIndex.ts`, `resources/indexes/hnswPlaneBinding.ts`)](resources/indexes/DESIGN.md#native-hnsw-plane-a-file-primary-mmap-graph-on-the-derived-index-runtime-resourcesindexeshierarchicalnavigablesmallworldts-resourcesindexeshnswderivedindexts-resourcesindexeshnswplanebindingts) — The mmap file owned by `@harperfast/hnsw` is the index; it is the default for audited RocksDB tables, and `nativePlane: false` keeps the JS graph. - [Scheduler: cluster-once execution without a consensus primitive (`resources/scheduler/`)](resources/scheduler/DESIGN.md#scheduler-cluster-once-execution-without-a-consensus-primitive-resourcesscheduler) — No consensus primitive exists; cluster-once relies on sticky leadership, heartbeat takeover and LWW lease convergence. ## resources/analytics/ +- [Full-text collection follows actual handle ownership](resources/analytics/DESIGN.md#full-text-collection-follows-actual-handle-ownership) — Opt-in. - [Analytics aggregation resumes from a raw cursor, which is not a clock (`resources/analytics/write.ts`)](resources/analytics/DESIGN.md#analytics-aggregation-resumes-from-a-raw-cursor-which-is-not-a-clock-resourcesanalyticswritets) — `rawCursor` moves only to a record a cycle actually read and is separate from the cadence marker. - [Per-thread rows carry a `thread` attribute under their own metric name (`resources/analytics/write.ts`)](resources/analytics/DESIGN.md#per-thread-rows-carry-a-thread-attribute-under-their-own-metric-name-resourcesanalyticswritets) — `perThread` entries roll up one row per thread; `duration` per thread is `duration-by-thread`, never extra `duration` rows; rows stay flat scalars for the Grafana datasource. · [A reset ends the window that was being flushed too (`resources/analytics/write.ts`)](resources/analytics/DESIGN.md#a-reset-ends-the-window-that-was-being-flushed-too-resourcesanalyticswritets) — `setAnalyticsEnabled()` advances a generation; a flush that sees it change stops delivering. - [Aggregation folds each measure over only the samples that carry it (`resources/analytics/write.ts`)](resources/analytics/DESIGN.md#aggregation-folds-each-measure-over-only-the-samples-that-carry-it-resourcesanalyticswritets) — per-measure sample counts, cycle-local, weight each mean; a measure in every sample folds exactly as before. diff --git a/benchmarks/fulltext-analytics.bench.js b/benchmarks/fulltext-analytics.bench.js new file mode 100644 index 0000000000..22a019982a --- /dev/null +++ b/benchmarks/fulltext-analytics.bench.js @@ -0,0 +1,279 @@ +require('../unitTests/testUtils'); +const assert = require('node:assert'); +const { readFileSync, writeFileSync } = require('node:fs'); +const { join } = require('node:path'); +const { setTimeout: delay } = require('node:timers/promises'); +const { monitorEventLoopDelay, performance } = require('node:perf_hooks'); +const { setupTestDBPath, ensureSystemTables } = require('../unitTests/testUtils'); +const { setRootConfigEntry, readRootConfig } = require('../unitTests/rootConfigFixture'); +const { getConfigFilePath } = require('#src/config/configUtils'); +const { table, getDatabases } = require('#src/resources/databases'); +const { fullTextDerivedIndexReadiness } = require('#src/resources/derivedIndexes'); +const { setMainIsWorker } = require('#src/server/threads/manageThreads'); +const { loadFullTextNativeBinding } = require('#src/resources/indexes/fullTextNativeBinding'); +const { + fullTextTelemetryCollector, + initializeFullTextTelemetry, +} = require('#src/resources/analytics/fullTextTelemetryRuntime'); +const analytics = require('#src/resources/analytics/write'); +const { waitFor } = require('../unitTests/waitFor'); + +const configuration = JSON.parse(process.env.FULLTEXT_ANALYTICS_BENCHMARK ?? 'null'); +if (!configuration) throw new Error('Run benchmarks/fulltext-analytics.mjs to create an isolated benchmark child'); + +it(`qualifies actual Harper fulltext analytics (${configuration.profile})`, async function () { + this.timeout(configuration.duration + 120_000); + setupTestDBPath(); + setMainIsWorker(true); + await ensureSystemTables(); + const binding = await loadFullTextNativeBinding(); + const runtime = await binding.runtimeInfo(); + assert.strictEqual(runtime.packageVersion, '0.9.0'); + const path = getConfigFilePath(); + const savedConfig = readFileSync(path, 'utf8'); + const products = []; + let removeListener, rssTimer, eventLoop; + let measuring = false; + let exportedRows = 0; + const observedSamples = new Map(); + let distinctObservedSamples = 0; + let result; + try { + initializeFullTextTelemetry(); + analytics.setAnalyticsEnabled(true); + setRootConfigEntry('analytics', { + ...readRootConfig().analytics, + fullText: { + enabled: false, + intervalMilliseconds: configuration.interval, + histograms: false, + snapshotBytes: false, + }, + }); + await waitFor(() => !fullTextTelemetryCollector.collecting); + for (let index = 0; index < configuration.indexes; index++) { + const Product = table({ + database: `analytics-benchmark-${process.pid}-${index}`, + table: 'Product', + audit: true, + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'title', type: 'String' }, + ], + fullTextIndexes: [ + { + name: 'catalogSearch', + fields: [{ name: 'title', weight: 1 }], + analyzer: 'english@2', + stopWords: true, + positions: true, + surfaceTerms: true, + synonyms: [], + }, + ], + }); + products.push(Product); + await Product.put(Array.from({ length: 32 }, (_, id) => ({ id: String(id), title: 'stable catalog widget' }))); + await waitFor(() => fullTextDerivedIndexReadiness(Product, 'catalogSearch').state === 'ready', 30_000); + } + const query = { + conditions: [ + { + attribute: 'catalogSearch', + comparator: 'matches', + value: 'widget', + maxIndexLagMilliseconds: 0, + waitForIndexMilliseconds: 5000, + }, + ], + limit: 4, + }; + const recordDuration = analytics.actionRecorder('duration', undefined, 'fulltext-benchmark', 'operation'); + async function search(Product) { + const start = performance.now(); + const rows = []; + for await (const row of await Product.search(query)) rows.push(await row); + const duration = performance.now() - start; + recordDuration(duration); + assert.strictEqual(rows.length, 4); + assert.strictEqual(new Set(rows.map((row) => row.id)).size, 4); + for (const row of rows) assert.ok(row.title.includes('widget')); + return duration; + } + async function load(duration, latencies) { + const deadline = performance.now() + duration; + let correctQueries = 0, + mutations = 0; + let failed = false; + const queries = Array.from({ length: 4 }, async (_, worker) => { + let sequence = worker; + while (!failed && performance.now() < deadline) { + const latency = await search(products[sequence++ % products.length]); + if (latencies) latencies[correctQueries % latencies.length] = latency; + correctQueries++; + } + }); + const writer = (async () => { + while (!failed && performance.now() < deadline) { + await products[mutations % products.length].put({ + id: '0', + title: `stable catalog widget revision ${mutations}`, + }); + mutations++; + await delay(50); + } + })(); + const settlements = await Promise.allSettled( + [...queries, writer].map((operation) => + operation.catch((error) => { + failed = true; + throw error; + }) + ) + ); + const failures = settlements.filter((entry) => entry.status === 'rejected').map((entry) => entry.reason); + if (failures.length) throw new AggregateError(failures, 'Fulltext benchmark workload failed'); + return { correctQueries, mutations }; + } + await load(2000); + setRootConfigEntry('analytics', { + ...readRootConfig().analytics, + fullText: { + enabled: configuration.profile === 'enabled', + intervalMilliseconds: configuration.interval, + histograms: false, + snapshotBytes: false, + }, + }); + await waitFor(() => fullTextTelemetryCollector.collecting === (configuration.profile === 'enabled')); + removeListener = analytics.addAnalyticsListener((metrics) => { + if (!measuring) return; + for (const row of metrics) + if (row.metric === 'fulltext') { + exportedRows++; + const key = `${row.path}/${row.method}`; + const identity = `${row.series}/${row.sampledAt}`; + if (observedSamples.get(key) !== identity) { + distinctObservedSamples++; + observedSamples.set(key, identity); + } + } + }); + const latencies = new Float64Array(100_000); + const rssStart = process.memoryUsage().rss; + let rssPeak = rssStart; + rssTimer = setInterval(() => { + rssPeak = Math.max(rssPeak, process.memoryUsage().rss); + }, 100).unref(); + eventLoop = monitorEventLoopDelay({ resolution: 20 }); + eventLoop.enable(); + const utilizationStart = performance.eventLoopUtilization(); + const cpuStart = process.cpuUsage(); + const started = performance.now(); + const startedWall = Date.now(); + measuring = true; + const counts = await load(configuration.duration, latencies); + measuring = false; + const elapsedMilliseconds = performance.now() - started; + const endedWall = Date.now(); + const cpu = process.cpuUsage(cpuStart); + const utilization = performance.eventLoopUtilization(utilizationStart); + const rssEnd = process.memoryUsage().rss; + rssPeak = Math.max(rssPeak, rssEnd); + clearInterval(rssTimer); + eventLoop.disable(); + const sorted = latencies.subarray(0, Math.min(counts.correctQueries, latencies.length)).slice().sort(); + const percentile = (fraction) => sorted[Math.max(0, Math.ceil(sorted.length * fraction) - 1)]; + assert.ok(counts.correctQueries > 0 && counts.mutations > 0); + if (configuration.profile === 'disabled') assert.strictEqual(exportedRows, 0); + const sufficient = configuration.duration >= Math.max(60_000, configuration.interval + 1000); + if (configuration.profile === 'enabled' && sufficient) + assert.ok(exportedRows > 0, 'enabled profile must traverse existing analytics flush/export'); + const persistedRows = () => { + let count = 0; + const store = getDatabases().system?.hdb_raw_analytics?.primaryStore; + if (store) + for (const { value } of store.getRange({ start: false, end: Infinity })) + if (value.time >= startedWall && value.time <= endedWall) + count += value.metrics.filter((row) => row.metric === 'fulltext').length; + return count; + }; + if (configuration.profile === 'enabled' && sufficient) await waitFor(() => persistedRows() > 0, 5000); + const persistedFullTextRawRows = persistedRows(); + if (configuration.profile === 'disabled') assert.strictEqual(persistedFullTextRawRows, 0); + result = { + profile: configuration.profile, + pair: configuration.pair, + source: configuration.source, + node: process.version, + platform: process.platform, + architecture: process.arch, + fulltext: runtime.packageVersion, + indexes: products.length, + queryConcurrency: 4, + writerDelayMilliseconds: 50, + configuredDurationMilliseconds: configuration.duration, + configuredIntervalMilliseconds: configuration.interval, + histograms: false, + snapshotBytes: false, + startupWarmupMilliseconds: 2000, + ...counts, + elapsedMilliseconds, + queriesPerSecond: counts.correctQueries / (elapsedMilliseconds / 1000), + throughputIncludesOracleAndCommonRequestAnalytics: true, + latencyIncludesOracle: false, + latencyRingCapacity: latencies.length, + latencySampleCount: sorted.length, + latencyCoverage: counts.correctQueries > latencies.length ? 'last-100000-completions' : 'all-completions', + latencyMilliseconds: { p50: percentile(0.5), p95: percentile(0.95), p99: percentile(0.99) }, + cpu, + eventLoop: { + utilization: utilization.utilization, + meanDelayMilliseconds: eventLoop.mean / 1e6, + maxDelayMilliseconds: eventLoop.max / 1e6, + p99DelayMilliseconds: eventLoop.percentile(99) / 1e6, + }, + rssBytes: { start: rssStart, end: rssEnd, peak: rssPeak }, + fullTextRowsExported: exportedRows, + distinctObservedSamples, + persistedFullTextRawRows, + rssSamplingIntervalMilliseconds: 100, + qualificationSufficient: sufficient, + scope: sufficient + ? 'finite local paired integration workload; not sustained production qualification' + : 'short smoke; insufficient full-cadence qualification', + }; + } catch (error) { + writeFileSync( + join(configuration.output, 'failure.json'), + JSON.stringify({ name: error.name, message: error.message, stack: error.stack }, null, 2) + ); + throw error; + } finally { + measuring = false; + clearInterval(rssTimer); + eventLoop?.disable(); + removeListener?.(); + const settlements = await Promise.allSettled( + products.map(async (Product) => { + const runtime = Product.derivedIndexRuntime; + if (runtime) await runtime.close(); + Product.derivedIndexRuntime = undefined; + }) + ); + writeFileSync(path, savedConfig); + const failures = settlements.filter((entry) => entry.status === 'rejected').map((entry) => entry.reason); + if (failures.length) { + writeFileSync( + join(configuration.output, 'cleanup-failure.json'), + JSON.stringify( + failures.map((error) => ({ name: error.name, message: error.message })), + null, + 2 + ) + ); + throw new AggregateError(failures, 'Fulltext qualification handles did not close cleanly'); + } + } + writeFileSync(join(configuration.output, 'result.json'), JSON.stringify({ ...result, cleanClose: true }, null, 2)); +}); diff --git a/benchmarks/fulltext-analytics.mjs b/benchmarks/fulltext-analytics.mjs new file mode 100644 index 0000000000..52c8e3cf08 --- /dev/null +++ b/benchmarks/fulltext-analytics.mjs @@ -0,0 +1,130 @@ +import { spawn } from 'node:child_process'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, writeFileSync, readFileSync, createWriteStream, existsSync, readdirSync } from 'node:fs'; +import { resolve, dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createHash } from 'node:crypto'; + +const repository = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const options = { duration: 70_000, indexes: 4, pairs: 2, interval: 60_000, output: undefined }; +const bounds = { duration: [100, 3_600_000], indexes: [1, 16], pairs: [1, 10], interval: [1000, 2_147_483_647] }; +for (let i = 2; i < process.argv.length; i += 2) { + if (!process.argv[i].startsWith('--')) throw new Error('Benchmark options must use --name value'); + const name = process.argv[i].slice(2); + const value = process.argv[i + 1]; + if (!Object.hasOwn(options, name) || value === undefined) + throw new Error( + 'Use --duration milliseconds --indexes 1..16 --pairs 1..10 --interval milliseconds --output directory' + ); + if (name === 'output') options.output = resolve(value); + else { + const number = Number(value); + if (!/^\d+$/.test(value) || !Number.isSafeInteger(number) || number < bounds[name][0] || number > bounds[name][1]) + throw new Error(`Invalid --${name}: expected integer ${bounds[name].join('..')}`); + options[name] = number; + } +} +options.output ??= `/Users/kyle/WebstormProjects/tmp/fulltext-analytics-${new Date().toISOString().replace(/[:.]/g, '-')}`; +if (existsSync(options.output) && readdirSync(options.output).length) + throw new Error('Use a fresh or empty --output directory to preserve prior qualification evidence'); +mkdirSync(options.output, { recursive: true }); +const git = (...args) => execFileSync('git', args, { cwd: repository, encoding: 'utf8' }).trim(); +const source = { + head: git('rev-parse', 'HEAD'), + dirty: Boolean(git('status', '--porcelain')), + trackedDiffSha256: createHash('sha256') + .update(git('diff', 'HEAD', '--', '.')) + .digest('hex'), +}; +const fingerprintFiles = [ + 'benchmarks/fulltext-analytics.mjs', + 'benchmarks/fulltext-analytics.bench.js', + 'resources/analytics/fullTextTelemetry.ts', + 'resources/analytics/fullTextTelemetryRuntime.ts', + 'config/fullTextTelemetryConfiguration.ts', +]; +source.fileSha256 = Object.fromEntries( + fingerprintFiles.map((file) => [ + file, + createHash('sha256') + .update(readFileSync(join(repository, file))) + .digest('hex'), + ]) +); +writeFileSync( + join(options.output, 'manifest.json'), + JSON.stringify( + { options, source, node: process.version, platform: process.platform, architecture: process.arch }, + null, + 2 + ) +); + +async function spawnMocha(profile, pair) { + const directory = join(options.output, `pair-${pair}-${profile}`); + mkdirSync(directory, { recursive: true }); + const configuration = { ...options, profile, pair, output: directory, source }; + writeFileSync(join(directory, 'manifest.json'), JSON.stringify(configuration, null, 2)); + const log = createWriteStream(join(directory, 'child.log')); + const childEnv = { + ...process.env, + FULLTEXT_ANALYTICS_BENCHMARK: JSON.stringify(configuration), + HARPER_STORAGE_ENGINE: 'rocksdb', + }; + for (const key of ['HARPER_CONFIG', 'HARPER_DEFAULT_CONFIG', 'HARPER_SET_CONFIG']) delete childEnv[key]; + const child = spawn( + process.execPath, + [ + join(repository, 'node_modules/mocha/bin/mocha.js'), + 'benchmarks/fulltext-analytics.bench.js', + '--timeout', + String(options.duration + 120_000), + '--forbid-pending', + '--fail-zero', + ], + { cwd: repository, env: childEnv, stdio: ['ignore', 'pipe', 'pipe'] } + ); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + child.kill('SIGTERM'); + }, options.duration + 120_000); + const result = await new Promise((resolveChild, reject) => { + child.once('error', reject); + child.once('exit', (code, signal) => resolveChild({ code, signal })); + }).finally(() => { + clearTimeout(timer); + log.end(); + }); + writeFileSync(join(directory, 'launcher-result.json'), JSON.stringify({ ...result, timedOut }, null, 2)); + if (result.code !== 0 || timedOut || !existsSync(join(directory, 'result.json'))) + throw new Error(`${profile} pair ${pair} failed; retained ${directory}`); + return JSON.parse(readFileSync(join(directory, 'result.json'), 'utf8')); +} + +const results = []; +for (let pair = 1; pair <= options.pairs; pair++) { + const profiles = pair % 2 ? ['disabled', 'enabled'] : ['enabled', 'disabled']; + for (const profile of profiles) { + process.stdout.write(`Pair ${pair}/${options.pairs}: ${profile}\n`); + results.push(await spawnMocha(profile, pair)); + writeFileSync(join(options.output, 'results.json'), JSON.stringify(results, null, 2)); + } +} +const comparisons = Array.from({ length: options.pairs }, (_, index) => { + const disabled = results.find((result) => result.pair === index + 1 && result.profile === 'disabled'); + const enabled = results.find((result) => result.pair === index + 1 && result.profile === 'enabled'); + return { + pair: index + 1, + throughputRatioEnabledToDisabled: enabled.queriesPerSecond / disabled.queriesPerSecond, + p99RatioEnabledToDisabled: enabled.latencyMilliseconds.p99 / disabled.latencyMilliseconds.p99, + cpuMicrosecondsPerQueryDisabled: (disabled.cpu.user + disabled.cpu.system) / disabled.correctQueries, + cpuMicrosecondsPerQueryEnabled: (enabled.cpu.user + enabled.cpu.system) / enabled.correctQueries, + enabledExportedRows: enabled.fullTextRowsExported, + qualificationSufficient: enabled.qualificationSufficient && disabled.qualificationSufficient, + }; +}); +writeFileSync(join(options.output, 'comparison.json'), JSON.stringify(comparisons, null, 2)); +process.stdout.write(`${JSON.stringify(comparisons, null, 2)}\nArtifacts: ${options.output}\n`); diff --git a/config-root.schema.json b/config-root.schema.json index 3ca49dfff6..4ead35cc22 100644 --- a/config-root.schema.json +++ b/config-root.schema.json @@ -1010,6 +1010,28 @@ "type": "object", "additionalProperties": false, "properties": { + "fullText": { + "type": "object", + "additionalProperties": false, + "description": "Live, opt-in native full-text analytics. Omitted means disabled; no polling. Counters always record in the wrapper.", + "properties": { + "enabled": { "type": "boolean" }, + "intervalMilliseconds": { + "type": "integer", + "minimum": 1000, + "maximum": 2147483647, + "description": "Per-handle collection interval. Implied default: 60000 ms. Calls are staggered." + }, + "histograms": { + "type": "boolean", + "description": "Independently opt in to histogram recording. Implied default: false." + }, + "snapshotBytes": { + "type": "boolean", + "description": "Independently opt in to reader byte observation. Implied default: false." + } + } + }, "aggregatePeriod": { "type": "number", "description": "Seconds between aggregation of raw analytics. Default: 60" diff --git a/config/RootConfigWatcher.ts b/config/RootConfigWatcher.ts index d409e2bfbe..c3b05f9dcf 100644 --- a/config/RootConfigWatcher.ts +++ b/config/RootConfigWatcher.ts @@ -64,6 +64,7 @@ export class RootConfigWatcher extends EventEmitter { // opens it without claiming the watch is armed, so the arming re-read still runs afterwards. #barrierOpen: boolean = false; #configLoaded: boolean = false; + #unavailable: boolean = false; #readyStaged: boolean = false; #readyEmitted: boolean = false; ready: Promise; @@ -88,6 +89,15 @@ export class RootConfigWatcher extends EventEmitter { }) .on('add', this.handleChange.bind(this)) .on('change', this.handleChange.bind(this)) + .on('unlink', () => { + if (this.#closed) return; + try { + this.#unavailable = true; + this.emit('unavailable'); + } catch (error) { + logger().warn('A Harper configuration availability listener failed', errorForLog(error)); + } + }) .on('error', this.handleError.bind(this)) // Generation-bound: `#armGate.reset()` runs before the failed watcher is closed, so a // `ready` still queued on it would arm the gate on the replacement's behalf and the @@ -219,7 +229,20 @@ export class RootConfigWatcher extends EventEmitter { } handleChange() { - if (this.#read(true)) this.#scheduleTrailingRead(); + if (this.#read(true)) { + this.#scheduleTrailingRead(); + this.#markAvailable(); + } + } + + #markAvailable() { + if (!this.#unavailable || this.#closed) return; + this.#unavailable = false; + try { + this.emit('available'); + } catch (error) { + logger().warn('A Harper configuration availability listener failed', errorForLog(error)); + } } #scheduleTrailingRead() { @@ -245,7 +268,10 @@ export class RootConfigWatcher extends EventEmitter { #schedule(): boolean { this.#clearTrailingRead(); return this.#readRetry.schedule(() => { - if (this.#read(false)) this.#scheduleTrailingRead(); + if (this.#read(false)) { + this.#scheduleTrailingRead(); + this.#markAvailable(); + } }, !this.#readyEmitted); } diff --git a/config/fullTextTelemetryConfiguration.ts b/config/fullTextTelemetryConfiguration.ts new file mode 100644 index 0000000000..0df2fa3813 --- /dev/null +++ b/config/fullTextTelemetryConfiguration.ts @@ -0,0 +1,68 @@ +import { isDeepStrictEqual } from 'node:util'; +import { getSharedRootConfigWatcher, type RootConfigWatcher } from './RootConfigWatcher.ts'; +import { overlayRootEnvConfig } from './harperConfigEnvVars.ts'; +import { get, refreshConfigSubtree } from '../utility/environment/environmentManager.ts'; +import { resolveFullTextTelemetryPolicy } from '../resources/analytics/fullTextTelemetry.ts'; +import { loggerWithTag } from '../utility/logging/logger.ts'; + +const logger = loggerWithTag('fulltext-telemetry'); +const PARAM = 'analytics_fullText'; + +/** Observe only telemetry settings; invalid edits retain the last valid policy. */ +export function watchFullTextTelemetryConfiguration( + apply: (value: unknown) => void, + watcher: RootConfigWatcher = getSharedRootConfigWatcher() +): () => void { + let stopped = false; + let previousRoot: unknown; + let lastPolicy: ReturnType | undefined; + function applyPolicy(value: unknown) { + const policy = resolveFullTextTelemetryPolicy(value); + if (isDeepStrictEqual(policy, lastPolicy)) return; + apply(policy); + lastPolicy = policy; + } + function report() { + try { + logger.warn('Could not apply full-text telemetry configuration; retaining the previous valid policy'); + } catch {} + } + function changed() { + if (stopped) return; + try { + const root = structuredClone(overlayRootEnvConfig(watcher.config) ?? {}); + const policy = refreshConfigSubtree(PARAM, previousRoot, root, resolveFullTextTelemetryPolicy); + applyPolicy(policy); + previousRoot = root; + } catch { + report(); + } + } + function unavailable() { + if (stopped) return; + try { + applyPolicy(refreshConfigSubtree(PARAM, previousRoot, {}, resolveFullTextTelemetryPolicy)); + previousRoot = {}; + } catch { + report(); + } + } + function stop() { + stopped = true; + watcher.off('change', changed); + watcher.off('unavailable', unavailable); + watcher.off('available', changed); + watcher.off('close', stop); + } + try { + applyPolicy(get(PARAM)); + } catch { + report(); + } + watcher.on('change', changed); + watcher.on('unavailable', unavailable); + watcher.on('available', changed); + watcher.on('close', stop); + void watcher.ready.then(changed).catch(report); + return stop; +} diff --git a/dependencies.md b/dependencies.md index b5e1f0e935..bd33b562bc 100644 --- a/dependencies.md +++ b/dependencies.md @@ -290,10 +290,10 @@ This is the inverse of the entries below — a dependency we take deliberate ste ## @harperfast/fulltext (optional dependency) - Need for usage: Supplies the Tantivy-backed native index used by Harper full-text derived indexes. Harper remains the source of truth; each node builds and advances its local index from committed records through the shared derived-index runtime. -- Size/memory cost: The root package is about 700 KB installed for 0.8.0, plus one platform-specific native package (about 7.0 MB on macOS arm64). Runtime memory and native index size depend on the indexed text, analyzer, and stored-field options. +- Size/memory cost: The exact-pinned 0.9.0 root package includes JavaScript facades and types plus one platform-specific native package. Runtime memory and native index size depend on the indexed text, analyzer, and stored-field options. - Security: First-party Apache-2.0 Harper package wrapping Tantivy. It runs native code in-process, so Harper exact-pins the root package and the package exact-pins every platform prebuild to the same version. - Environment interaction: Lazily loaded only when a full-text index is activated. It creates Tantivy index files in Harper's derived-index directory; those files are disposable local state and are rebuilt or replayed from Harper records after loss or incompatibility. Upgrading the 0.5.0 engine shipped in Harper 5.3.4 requires a full local rebuild on every node; see the [deployment and recovery guidance](resources/FULL_TEXT.md#upgrading-an-existing-native-index). -- Overlap: None. Harper owns schema, transactions, replication, source-record reads, and derived-index coordination; the package owns native indexing, persistence, and search execution. +- Overlap: None. Harper owns schema, transactions, replication, source-record reads, derived-index coordination and opt-in analytics scheduling/export; the package owns native indexing, persistence, search execution and native/facade telemetry accounting. Harper requires native ABI 14, query API 8 and telemetry API 1. - Transitive dependencies: Only exact-version, platform-specific optional prebuild packages; no JavaScript runtime dependency tree. - Binary compilation: Supported Linux glibc x64/arm64, macOS arm64, and Windows x64 targets use prebuilds. Because the root package is optional, Harper still installs if a native package is unavailable, but a declared full-text index cannot activate until the module is installed. - Can be deferred: Yes. The module is not loaded for tables without full-text indexes. @@ -311,3 +311,9 @@ This is the inverse of the entries below — a dependency we take deliberate ste - Need for usage: Exact npm alias for fulltext 0.7.0, used by the existing isolated upgrade fixture to write a real text-only checkpoint and verify 0.8 reuse and subsequent 0.7 reopen. The separate 0.5.0 alias retains coverage of the earlier incompatible engine. - Updates and removal: Renovate leaves this historical alias fixed. Remove it and its exclusion only when the 0.7 migration fixture is retired. - Cost and boundary: One additional root package and supported platform prebuild in development/CI only. No production runtime import; the previous-engine child exits before the current engine opens its files. + +## @harperfast/fulltext-v08 (dev dependency) + +- Need for usage: Exact npm alias for fulltext 0.8.0. The isolated migration fixture writes a genuine prior-release checkpoint, verifies current reuse without rebuilding, advances it and checks the prior release can reopen it. +- Cost and boundary: One additional root package and supported platform prebuild in development/CI only; never imported by production code. +- Updates and removal: Keep the historical alias exact-pinned and excluded from Renovate updates. Remove it with its fixture when migration from 0.8 is no longer supported. diff --git a/package-lock.json b/package-lock.json index 7b29cfdf6d..5189d86d6f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -102,6 +102,7 @@ "@harperfast/code-guidelines": "^0.1.2", "@harperfast/fulltext-previous": "npm:@harperfast/fulltext@0.5.0", "@harperfast/fulltext-v07": "npm:@harperfast/fulltext@0.7.0", + "@harperfast/fulltext-v08": "npm:@harperfast/fulltext@0.8.0", "@harperfast/integration-testing": "^1.0.0", "@modelcontextprotocol/sdk": "^1.29.0", "@types/busboy": "^1.5.4", @@ -147,7 +148,7 @@ "node": "^22.18.0 || >=24.0.0" }, "optionalDependencies": { - "@harperfast/fulltext": "0.8.0", + "@harperfast/fulltext": "0.9.0", "@harperfast/hnsw": "0.4.0", "bufferutil": "4.1.0", "segfault-handler": "1.3.0" @@ -2411,25 +2412,25 @@ "license": "Apache-2.0" }, "node_modules/@harperfast/fulltext": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext/-/fulltext-0.8.0.tgz", - "integrity": "sha512-7T+i2PbdsyrXwb4V2mcE23Pk8DwOl12vgE8YodTe/KUeS7tzJp8I/uQOkD5YiW1ga4WasfLVbZrcTi5pULMjAg==", + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext/-/fulltext-0.9.0.tgz", + "integrity": "sha512-FOM5UyIIjNZ4ieUDj6gJTqKTwmsm+DGLNZ0+W9KJHKJKbn5jZGV7A0MHhIxPBouOuBXLFx4HR1wqyt5VeIGqOw==", "license": "Apache-2.0", "optional": true, "engines": { "node": "^22.18.0 || >=24.0.0" }, "optionalDependencies": { - "@harperfast/fulltext-darwin-arm64": "0.8.0", - "@harperfast/fulltext-linux-arm64-gnu": "0.8.0", - "@harperfast/fulltext-linux-x64-gnu": "0.8.0", - "@harperfast/fulltext-win32-x64-msvc": "0.8.0" + "@harperfast/fulltext-darwin-arm64": "0.9.0", + "@harperfast/fulltext-linux-arm64-gnu": "0.9.0", + "@harperfast/fulltext-linux-x64-gnu": "0.9.0", + "@harperfast/fulltext-win32-x64-msvc": "0.9.0" } }, "node_modules/@harperfast/fulltext-darwin-arm64": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-darwin-arm64/-/fulltext-darwin-arm64-0.8.0.tgz", - "integrity": "sha512-Egmt0FsOqgTEeZo6YpxMbywTbaO8aH3gDqHpC+lfmrqXAFiDub8NqQ72xSAI0VUfSMXUL83/NoHGX68brZ+upA==", + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-darwin-arm64/-/fulltext-darwin-arm64-0.9.0.tgz", + "integrity": "sha512-NGV6NKFwasLrf6usY36G/CvSaDClhsMQ63BAwkcSg5iqlMPp1UYqED3MnANLedwvgnW43yz4jb6evNB+7jaUUg==", "cpu": [ "arm64" ], @@ -2443,9 +2444,9 @@ } }, "node_modules/@harperfast/fulltext-linux-arm64-gnu": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-arm64-gnu/-/fulltext-linux-arm64-gnu-0.8.0.tgz", - "integrity": "sha512-YfLa+bjnZUXI5bY3JPOmlkn3DlKXA3zheryUxD+IKTjUvR5ekNoYbl5WTEzYhqRZGZs2o07ct2+Zb3otiUP3Ww==", + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-arm64-gnu/-/fulltext-linux-arm64-gnu-0.9.0.tgz", + "integrity": "sha512-YICMovXLg94eAYfw+2Z2iaQiOem+GbbAuAiFw4x6D9rjqysppqI2Go/zy66N5jHpxO1o7mOiZdOWk67m+oikBg==", "cpu": [ "arm64" ], @@ -2462,9 +2463,9 @@ } }, "node_modules/@harperfast/fulltext-linux-x64-gnu": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-x64-gnu/-/fulltext-linux-x64-gnu-0.8.0.tgz", - "integrity": "sha512-xLTpOAyNax0FpKKoMsnIRgcyMD1sdCVhKmBXO5pMPB9vHoAoTBoeyGjKxAv2HGqwcOXT5sfhAGmCUpqPlbDtVA==", + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-x64-gnu/-/fulltext-linux-x64-gnu-0.9.0.tgz", + "integrity": "sha512-i7JSYXgN6gDkrk/1X5STrHBHs/2hvnGJ8XKw9h1H/yKDcyVsKo8pNmneV7NoHB4468gqYiu6CbRk7aw5KK7XUg==", "cpu": [ "x64" ], @@ -2662,13 +2663,104 @@ "node": "^22.18.0 || >=24.0.0" } }, - "node_modules/@harperfast/fulltext-win32-x64-msvc": { + "node_modules/@harperfast/fulltext-v08": { + "name": "@harperfast/fulltext", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext/-/fulltext-0.8.0.tgz", + "integrity": "sha512-7T+i2PbdsyrXwb4V2mcE23Pk8DwOl12vgE8YodTe/KUeS7tzJp8I/uQOkD5YiW1ga4WasfLVbZrcTi5pULMjAg==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^22.18.0 || >=24.0.0" + }, + "optionalDependencies": { + "@harperfast/fulltext-darwin-arm64": "0.8.0", + "@harperfast/fulltext-linux-arm64-gnu": "0.8.0", + "@harperfast/fulltext-linux-x64-gnu": "0.8.0", + "@harperfast/fulltext-win32-x64-msvc": "0.8.0" + } + }, + "node_modules/@harperfast/fulltext-v08/node_modules/@harperfast/fulltext-darwin-arm64": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-darwin-arm64/-/fulltext-darwin-arm64-0.8.0.tgz", + "integrity": "sha512-Egmt0FsOqgTEeZo6YpxMbywTbaO8aH3gDqHpC+lfmrqXAFiDub8NqQ72xSAI0VUfSMXUL83/NoHGX68brZ+upA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/@harperfast/fulltext-v08/node_modules/@harperfast/fulltext-linux-arm64-gnu": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-arm64-gnu/-/fulltext-linux-arm64-gnu-0.8.0.tgz", + "integrity": "sha512-YfLa+bjnZUXI5bY3JPOmlkn3DlKXA3zheryUxD+IKTjUvR5ekNoYbl5WTEzYhqRZGZs2o07ct2+Zb3otiUP3Ww==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/@harperfast/fulltext-v08/node_modules/@harperfast/fulltext-linux-x64-gnu": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-x64-gnu/-/fulltext-linux-x64-gnu-0.8.0.tgz", + "integrity": "sha512-xLTpOAyNax0FpKKoMsnIRgcyMD1sdCVhKmBXO5pMPB9vHoAoTBoeyGjKxAv2HGqwcOXT5sfhAGmCUpqPlbDtVA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/@harperfast/fulltext-v08/node_modules/@harperfast/fulltext-win32-x64-msvc": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/@harperfast/fulltext-win32-x64-msvc/-/fulltext-win32-x64-msvc-0.8.0.tgz", "integrity": "sha512-GKQ9GR8WuN8T4tqZS+smQanLm97lMiA2uZv6pMJk79zh4xtNrC5wOzARx3/J3l3wclI2HCf+iVKgn1OxOfvR7w==", "cpu": [ "x64" ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/@harperfast/fulltext-win32-x64-msvc": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-win32-x64-msvc/-/fulltext-win32-x64-msvc-0.9.0.tgz", + "integrity": "sha512-sqyuyhN/W2DMtkkOCyWYfjbWFaRtzvIdDQT+r7mRRB5As0BZ4UJejN+Q+TChiOasA8J1j4nO3NE4l6fDw0lqyA==", + "cpu": [ + "x64" + ], "license": "Apache-2.0", "optional": true, "os": [ diff --git a/package.json b/package.json index 3b4f1b63d7..4252ec2a76 100644 --- a/package.json +++ b/package.json @@ -164,6 +164,7 @@ "@harperfast/code-guidelines": "^0.1.2", "@harperfast/fulltext-previous": "npm:@harperfast/fulltext@0.5.0", "@harperfast/fulltext-v07": "npm:@harperfast/fulltext@0.7.0", + "@harperfast/fulltext-v08": "npm:@harperfast/fulltext@0.8.0", "@harperfast/integration-testing": "^1.0.0", "@modelcontextprotocol/sdk": "^1.29.0", "@types/busboy": "^1.5.4", @@ -296,7 +297,7 @@ } }, "optionalDependencies": { - "@harperfast/fulltext": "0.8.0", + "@harperfast/fulltext": "0.9.0", "@harperfast/hnsw": "0.4.0", "bufferutil": "4.1.0", "segfault-handler": "1.3.0" diff --git a/renovate.json b/renovate.json index cbaf91ea8e..4ec897a830 100644 --- a/renovate.json +++ b/renovate.json @@ -53,10 +53,10 @@ "matchPackageNames": ["!alasql"] }, { - "description": "Keep the historical fulltext 0.5.0 and 0.7.0 upgrade fixtures fixed; remove this rule with the fixture alias.", + "description": "Keep the historical fulltext 0.5.0, 0.7.0 and 0.8.0 upgrade fixtures fixed; remove this rule with the fixture alias.", "matchManagers": ["npm"], "matchDepTypes": ["devDependencies"], - "matchDepNames": ["@harperfast/fulltext-previous", "@harperfast/fulltext-v07"], + "matchDepNames": ["@harperfast/fulltext-previous", "@harperfast/fulltext-v07", "@harperfast/fulltext-v08"], "enabled": false } ] diff --git a/resources/FULL_TEXT.md b/resources/FULL_TEXT.md index 701b73db81..0b7539932a 100644 --- a/resources/FULL_TEXT.md +++ b/resources/FULL_TEXT.md @@ -88,11 +88,11 @@ Weight and highlighting changes preserve the physical index and refresh query re ## Native package and limits -Harper pins `@harperfast/fulltext` to **0.8.0**, embedding **Tantivy 0.27.0**, and validates the native runtime capabilities before activation. Full-text activation requires RocksDB and an audited table. Declare new schema tables with `@table(audit: true)` and keep audit logging enabled while an index is declared. LMDB is rejected. A missing or incompatible native package prevents activation. +Harper pins `@harperfast/fulltext` to **0.9.0**, embedding **Tantivy 0.27.0**, and validates the native runtime capabilities before activation. Full-text activation requires RocksDB and an audited table. Declare new schema tables with `@table(audit: true)` and keep audit logging enabled while an index is declared. LMDB is rejected. A missing or incompatible native package prevents activation. ### Upgrading from fulltext 0.7.0 -Harper's existing indexes written by fulltext 0.7.0 contain text fields only and remain reusable with 0.8.0. This upgrade does not add native filter metadata or change their storage identity. Query limits and fuzzy controls do not require rebuilding. Stop all processes sharing a local index before switching package versions; never mix facade/native binaries or engine versions against a live directory. New installations omit `storage.fullText.query`; defaults apply automatically. An explicitly configured query block must be removed before returning to Harper 5.4.0-alpha.1 or alpha.2, whose validators reject it. Existing configuration is not rewritten. Returning to fulltext 0.7.0 after 0.8.0 has written to an index is not a qualified operator rollback procedure. +Harper's existing indexes written by fulltext 0.7.0 contain text fields only and remain reusable with 0.9.0. Genuine 0.8.0 checkpoints are also reused without rebuilding; the regression advances them with 0.9.0 and reopens them with 0.8.0. This binary-level check is not a supported older-Harper operator downgrade procedure. This upgrade does not add native filter metadata or change their storage identity. Query limits and fuzzy controls do not require rebuilding. Stop all processes sharing a local index before switching package versions; never mix facade/native binaries or engine versions against a live directory. New installations omit `storage.fullText.query`; defaults apply automatically. An explicitly configured query block must be removed before returning to Harper 5.4.0-alpha.1 or alpha.2, whose validators reject it. Existing configuration is not rewritten. Returning to fulltext 0.7.0 after 0.8.0 has written to an index is not a qualified operator rollback procedure. ### Upgrading an existing native index @@ -102,17 +102,17 @@ Drain a node's full-text traffic and stop its old Harper processes before starti Prerelease installations using fulltext 0.6 also require a full rebuild. -**Direct downgrade to fulltext 0.5.x is unsupported.** That engine refuses both opening and resetting a 0.8 identity, leaving full-text search unavailable. If a downgrade has already happened, keep search traffic drained and restore the 0.8.0-based Harper version. Harper does not yet provide a supported rollback procedure for full-text indexes; see [Implement full-text generation lifecycle, rebuild, eviction, backup, and status](https://github.com/HarperFast/harper/issues/2510). Do not edit, move or delete files in full-text index directories by hand. +**Direct downgrade to fulltext 0.5.x is unsupported.** That engine refuses both opening and resetting the current identity, leaving full-text search unavailable. If a downgrade has already happened, keep search traffic drained and restore the 0.9.0-based Harper version. Harper does not yet provide a supported rollback procedure for full-text indexes; see [Implement full-text generation lifecycle, rebuild, eviction, backup, and status](https://github.com/HarperFast/harper/issues/2510). Do not edit, move or delete files in full-text index directories by hand. ### Upgrade and recovery checklist 1. Drain the node's full-text requests and stop every old Harper process that shares its storage, including background workers. Do not mix engine versions against one local index. 2. Confirm authoritative records and their recovery backups are available. Reserve disk and time for a complete rebuild of every local full-text index; the tests do not establish a universal capacity or duration estimate. -3. Start the 0.8.0-based Harper version. Use `describe_table` for each affected table and check every `full_text_indexes[].readiness.state`. Keep full-text traffic drained while any index is not `ready`. +3. Start the 0.9.0-based Harper version. Use `describe_table` for each affected table and check every `full_text_indexes[].readiness.state`. Keep full-text traffic drained while any index is not `ready`. 4. Verify representative searches with `maxIndexLagMilliseconds: 0` and `waitForIndexMilliseconds: 30000`, so they wait for the writes committed before each search. Check the response body for stream errors as well as its HTTP status. Watch write-503 responses and retry with bounded backoff while replay catches up. 5. Restore full-text traffic only after all required indexes and current-coverage searches succeed. Readiness does not establish catalog-scale latency, spare disk capacity or coverage of writes committed after a search began. -If startup or rebuilding fails, keep the node's full-text traffic drained, preserve its source data and logs, and correct the reported package, storage or coordination failure before restarting the same 0.8.0-based version. An interrupted rebuild is recovered from authoritative records; do not force readiness or reset a live directory. After an accidental downgrade refusal, stop the older processes and restore the 0.8.0-based version before checking readiness and catch-up again. If that version also fails, stop and investigate rather than deleting index files. This is current-version recovery; an operator downgrade procedure remains unqualified. +If startup or rebuilding fails, keep the node's full-text traffic drained, preserve its source data and logs, and correct the reported package, storage or coordination failure before restarting the same 0.9.0-based version. An interrupted rebuild is recovered from authoritative records; do not force readiness or reset a live directory. After an accidental downgrade refusal, stop the older processes and restore the 0.9.0-based version before checking readiness and catch-up again. If that version also fails, stop and investigate rather than deleting index files. This is current-version recovery; an operator downgrade procedure remains unqualified. The upgrade and recovery checks cover bounded fixture workloads. Multi-node deployment, sustained-write rejection windows and workload capacity still require the qualification in [Implement full-text generation lifecycle, rebuild, eviction, backup, and status](https://github.com/HarperFast/harper/issues/2510) and [Qualify native full-text filtering and catalog-scale operation for v5.4](https://github.com/HarperFast/harper/issues/2512). @@ -192,3 +192,29 @@ Distance two remains preview and requires operator opt-in with `storage.fullText Metadata sorting, native structured-filter pushdown, candidate-ID pushdown and facets/nested aggregations are not enabled by this package upgrade. Their schema, authorization, consistency and workload qualification remain in [Qualify native full-text filtering and catalog-scale operation for v5.4](https://github.com/HarperFast/harper/issues/2512). For implementation details, see the [full-text invariants in the Resource design guide](DESIGN.md#full-text-declarations-and-reader-snapshots). The schema contract lives in [schema.graphql](../schema.graphql); compiler and generation rules live in [fullTextSchema.ts](fullTextSchema.ts). + +## Native full-text analytics + +Fulltext 0.9.0 requires native ABI 14, query API 8 and telemetry API 1. Deploy the exact facade and matching platform binary together. Harper rejects a missing or incompatible capability before activating an index, even when collection is disabled. + +Collection is off when `analytics.fullText` is absent. No telemetry polling timer, recurring scrape or full-text export runs by default. Native counters and timing totals still accumulate in the wrapper. Enable collection without restarting or reopening indexes: + +```yaml +analytics: + fullText: + enabled: true +``` + +The implied interval is 60,000 milliseconds per handle. Calls are staggered across existing writer/readers, and their rows use Harper's existing analytics flush, storage, retention and protected `get_analytics` interface. Collection never opens a cold index. To tune it, set `intervalMilliseconds` between 1,000 and 2,147,483,647; synchronous snapshots and export work have measurable cost, so qualify the intended number of indexes and cadence. No production overhead bound is claimed. + +Set `enabled: false` or remove the block to stop scheduled polling and persistent full-text export. A synchronous observation already executing completes. Editing unrelated settings does not restart collection. Invalid telemetry edits retain the last valid policy; deletion of the configuration file pauses operator collection until a valid file returns. Environment overlays retain their existing precedence. Disabling analytics also disables its full-text collection demand. Internal registered consumers can independently request observations; removing the last demand stops the timer. Consumer-only observations do not enable persistent analytics. + +`histograms: true` and `snapshotBytes: true` independently enable optional recording/reader-byte observation; both default to false. Turning collection off explicitly disables options no remaining consumer needs. Re-enabling resumes cumulative counters. Enabling bytes schedules no native publication: values can remain pending until a successful publication/reload opportunity. Cached bytes, observation age and errors retain their own meanings; reader bytes are not directory totals or resident memory. + +Read `get_analytics` with `metric: fulltext`. Each stored row identifies `path` (database/table/index), `method` (`writer` or `reader`), worker `thread`, a bounded-lifetime `series` reset marker and actual `sampledAt`. Field names flatten the wrapper snapshot: for example, `native_operations_search_completed`, `native_operations_search_queue_sumNanoseconds`, `native_queues_ordinarySearch_commands` and `logicalBatches_completed`. Values are flat JSON scalars; exact bigint counters, nanoseconds, bytes, histogram counts and bounds are decimal strings. Null and availability/pending/error flags mean unavailable, not zero. Histograms retain noncumulative buckets; no percentiles are invented. + +Native uint64 counter wraps are unwrapped into exact logical-series totals. Reader reopen and overlapping retired readers carry counters into the same worker/index/role series; document/segment/generation observations come from the most recently observed eligible reader rather than summing duplicate snapshots. Queue occupancy sums live handles of the same role; oldest age uses their maximum. Shared queue capacities overlap: never add ordinary/expensive capacities or treat them as a process-wide budget. Final close counters are sampled once after settlement; no recurring scrape runs while close is in flight. + +Rollups keep the newest cumulative row per worker/index/role instead of averaging counters. For rates, decode decimal counters with `BigInt`, require the same `series` and complete/available observations, subtract exact totals and divide by actual `sampledAt` elapsed time. Check integer safety before converting a delta to `number`. A new series, unavailable samples or incomplete coverage require a new baseline; never report a missing interval as zero activity. `complete: false` flags lost terminal coverage; `observationAgeMilliseconds` distinguishes retained observations from the latest attempt. Pending rows are coalesced per logical series in a 128-row buffer; overflow drops the oldest pending row and increments exact `droppedRows`. Historical rows follow configured analytics retention, not handle lifetime. + +Do not add facade logical batches to their native frame counts, sum overlapping legacy/native timing spans, or sum reader document counts across workers. Generation IDs are not source-log positions or freshness timestamps. Harper still owns readiness, source catch-up, authorization, hydration and end-to-end request attribution; this telemetry does not replace those checks. Queries, terms, record IDs, users, raw cursors, filesystem paths and raw exception messages are not exported labels. Detailed automatic merge statistics, cancellation and per-index RSS remain explicitly unsupported. diff --git a/resources/analytics/DESIGN.md b/resources/analytics/DESIGN.md index f164f571b6..62ff009505 100644 --- a/resources/analytics/DESIGN.md +++ b/resources/analytics/DESIGN.md @@ -70,3 +70,11 @@ the key's first sample lacked, or that only some threads carry, is kept. That ho thread record carries nothing but measures and its `count`, which the thread count then replaces: the fold keeps `total`/`ratio` on non-thread actions only, so a `ratio` on a thread record is always a caller's measure. + +## Full-text collection follows actual handle ownership + +`fullTextTelemetry.ts` keeps private per-handle baselines and carries exact counters into a stable worker/index/role series. `nativeFullTextTelemetry.test.js` pins reader churn, wrap/precision, opt-in toggles and final close. The scheduler pauses before native close and observes final scalars only after settlement, so it never polls close in flight. Completed registrations release handle references; inactive groups remain only until their coalesced final row is drained. Pending export has a 128-row memory bound with explicit drop counts. + +`fullTextTelemetryRuntime.ts` joins removable internal consumer demand with live operator policy. Only operator demand enables persistent analytics. No demand means no telemetry timer; no cold handles are opened. Counters are decimal strings, rows are flat and `cumulative` makes the aggregation fold retain the newest sample, including availability/error fields, per worker. Existing request metrics retain their averaging semantics. `aggregationCycle.test.js` verifies exact counters above 2^53 survive storage and `get_analytics`. + +`watchFullTextTelemetryConfiguration` uses the shared root watcher, environment overlay and transactional `refreshConfigSubtree`; unrelated edits preserve sampling cadence, invalid edits preserve valid policy, and availability events pause/recover after file deletion. `fullTextTelemetryConfiguration.test.js` names the effective override and recovery controls. Native runtime/query policy remains frozen separately. diff --git a/resources/analytics/fullTextTelemetry.ts b/resources/analytics/fullTextTelemetry.ts new file mode 100644 index 0000000000..1d0dc81e55 --- /dev/null +++ b/resources/analytics/fullTextTelemetry.ts @@ -0,0 +1,442 @@ +import { randomUUID } from 'node:crypto'; +import type { ConfigureTelemetryOptions, FullTextTelemetry } from '@harperfast/fulltext/native'; + +export type FullTextTelemetryPolicy = Required & { + enabled: boolean; + intervalMilliseconds: number; +}; + +export const DEFAULT_FULL_TEXT_TELEMETRY_POLICY: Readonly = Object.freeze({ + enabled: false, + intervalMilliseconds: 60_000, + histograms: false, + snapshotBytes: false, +}); + +export function resolveFullTextTelemetryPolicy(value: unknown): FullTextTelemetryPolicy { + if (value === undefined) return { ...DEFAULT_FULL_TEXT_TELEMETRY_POLICY }; + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new TypeError('analytics.fullText must be an object'); + const options = value as Record; + for (const key of Object.keys(options)) + if (!Object.hasOwn(DEFAULT_FULL_TEXT_TELEMETRY_POLICY, key)) + throw new TypeError(`Unknown analytics.fullText option '${key}'`); + const result = { ...DEFAULT_FULL_TEXT_TELEMETRY_POLICY, ...options } as FullTextTelemetryPolicy; + for (const key of ['enabled', 'histograms', 'snapshotBytes'] as const) + if (typeof result[key] !== 'boolean') throw new TypeError(`analytics.fullText.${key} must be boolean`); + if ( + !Number.isSafeInteger(result.intervalMilliseconds) || + result.intervalMilliseconds < 1_000 || + result.intervalMilliseconds > 2_147_483_647 + ) + throw new TypeError('analytics.fullText.intervalMilliseconds must be an integer between 1000 and 2147483647'); + return result; +} + +export interface FullTextTelemetryHandle { + telemetry(): FullTextTelemetry; + configureTelemetry(options: ConfigureTelemetryOptions): void; +} + +export type FullTextTelemetrySample = { + index: string; + role: 'writer' | 'reader'; + series: string; + sampledAt: number; + snapshot?: FullTextTelemetry; + error?: 'configuration-error' | 'collection-error'; + stage?: 'closing'; +}; + +type Scalar = string | number | boolean | null; +export type FullTextTelemetryRow = Record; +type FlatSnapshot = { counters: Map; gauges: FullTextTelemetryRow }; +type Group = { + index: string; + role: 'writer' | 'reader'; + series: string; + entries: Set; + totals: Map; + complete: boolean; +}; +type Entry = { + handle: FullTextTelemetryHandle; + group: Group; + previous: Map; + flat?: FlatSnapshot; + observedAt: number; + nextAt: number; + closing: boolean; + configured: boolean; + warned: boolean; +}; +type Consumer = { + callback: (sample: FullTextTelemetrySample) => void | Promise; + policy: FullTextTelemetryPolicy; + busy: boolean; +}; + +const UINT64_MODULUS = 1n << 64n; +export function fullTextNativeCounterDelta(current: bigint, previous: bigint): bigint { + return current >= previous ? current - previous : UINT64_MODULUS - previous + current; +} + +function isCounter(path: string[]): boolean { + if (path.some((part) => part.startsWith('upperBounds'))) return false; + if (path[0] === 'logicalBatches' || path[0] === 'preparationRejections') return true; + return path[0] === 'native' && (path[1] === 'phases' || (path[1] === 'operations' && path[3] !== 'dispatched')); +} + +export function flattenFullTextTelemetry(snapshot: FullTextTelemetry): FlatSnapshot { + const counters = new Map(); + const gauges: FullTextTelemetryRow = {}; + function visit(value: unknown, path: string[]) { + const name = path.join('_'); + if (typeof value === 'bigint') { + if (isCounter(path)) counters.set(name, (counters.get(name) ?? 0n) + value); + else gauges[name] = value.toString(); + } else if (value === null || typeof value === 'boolean' || typeof value === 'number') { + gauges[name] = value as Scalar; + } else if (typeof value === 'string') { + if (path.at(-1) === 'state') gauges[name] = value; + else if (path.at(-1)?.endsWith('Error') || path.at(-1) === 'error') + gauges[name] = /^E_[A-Z_]{1,60}$/.test(value) ? value : 'other'; + else gauges[name] = path.includes('unsupported') ? 'unsupported' : 'unavailable'; + } else if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) visit(value[i], [...path, String(i)]); + } else if (value && typeof value === 'object') { + for (const [key, child] of Object.entries(value)) { + const boundedKey = path.at(-1) === 'errors' && !/^E_[A-Z_]{1,60}$/.test(key) ? 'other' : key; + visit(child, [...path, boundedKey]); + } + } + } + visit(snapshot, []); + return { counters, gauges }; +} + +export class FullTextTelemetryCollector { + #groups = new Map(); + #entries = new Set(); + #registered = new WeakMap(); + #finished = new WeakSet(); + #consumers = new Set(); + #pending = new Map(); + #operator = { ...DEFAULT_FULL_TEXT_TELEMETRY_POLICY }; + #policy = { ...DEFAULT_FULL_TEXT_TELEMETRY_POLICY }; + #timer?: NodeJS.Timeout; + #revision = 0; + #stopped = false; + #maxPendingRows: number; + #droppedRows = 0n; + #onPending: () => void; + #onError: () => void; + + constructor(options: { onPending?: () => void; onError?: () => void; maxPendingRows?: number } = {}) { + this.#maxPendingRows = options.maxPendingRows ?? 128; + if (!Number.isSafeInteger(this.#maxPendingRows) || this.#maxPendingRows < 1) + throw new TypeError('The full-text telemetry pending-row budget must be a positive safe integer'); + this.#onPending = options.onPending ?? (() => {}); + this.#onError = options.onError ?? (() => {}); + } + + get collecting(): boolean { + return this.#policy.enabled && !this.#stopped; + } + + get scheduled(): boolean { + return this.#timer !== undefined; + } + + setOperatorPolicy(value: unknown): void { + this.#operator = resolveFullTextTelemetryPolicy(value); + if (!this.#operator.enabled) this.#pending.clear(); + this.#updatePolicy(); + } + + subscribe( + callback: (sample: FullTextTelemetrySample) => void | Promise, + options: Omit, 'enabled'> = {} + ): () => void { + if (typeof callback !== 'function') throw new TypeError('A full-text telemetry consumer must be a function'); + if (this.#stopped) throw new Error('Full-text telemetry collection has stopped'); + const consumer = { callback, policy: resolveFullTextTelemetryPolicy({ ...options, enabled: true }), busy: false }; + this.#consumers.add(consumer); + this.#updatePolicy(); + return () => { + if (this.#consumers.delete(consumer)) this.#updatePolicy(); + }; + } + + register(handle: FullTextTelemetryHandle, index: string, role: 'writer' | 'reader') { + if (!index || Buffer.byteLength(index) > 256 || (role !== 'writer' && role !== 'reader')) + throw new TypeError('Full-text telemetry requires a bounded index label and writer/reader role'); + const previous = this.#registered.get(handle); + if (previous) return this.#registration(previous); + if (this.#finished.has(handle) || this.#stopped) return { beforeClose() {}, afterClose() {} }; + const key = JSON.stringify([index, role]); + let group = this.#groups.get(key); + if (!group) { + group = { index, role, series: randomUUID(), entries: new Set(), totals: new Map(), complete: true }; + this.#groups.set(key, group); + } + const entry: Entry = { + handle, + group, + previous: new Map(), + observedAt: 0, + nextAt: performance.now() + this.#policy.intervalMilliseconds / (this.#entries.size + 1), + closing: false, + configured: false, + warned: false, + }; + group.entries.add(entry); + this.#entries.add(entry); + this.#registered.set(handle, entry); + if (this.collecting) this.#configure(entry); + this.#schedule(); + return this.#registration(entry); + } + + #registration(entry: Entry) { + return { + beforeClose: () => { + entry.closing = true; + if (this.collecting) this.#publish(entry, undefined, undefined, 'closing'); + this.#schedule(); + }, + afterClose: () => { + if (!this.#entries.has(entry)) return; + if (this.collecting) this.#observe(entry); + else entry.group.complete = false; + this.#entries.delete(entry); + this.#registered.delete(entry.handle); + this.#finished.add(entry.handle); + entry.group.entries.delete(entry); + this.#prune(); + this.#schedule(); + }, + }; + } + + collect(handle: object): void { + if (!this.collecting) return; + const entry = this.#registered.get(handle); + if (entry && !entry.closing) this.#observe(entry); + } + + beforeClose(handle: object): void { + const entry = this.#registered.get(handle); + if (entry) this.#registration(entry).beforeClose(); + } + + afterClose(handle: object): void { + const entry = this.#registered.get(handle); + if (entry) this.#registration(entry).afterClose(); + } + + takeRows(): FullTextTelemetryRow[] { + const rows = [...this.#pending.values()]; + this.#pending.clear(); + this.#prune(); + return rows; + } + + stop(): void { + if (this.#stopped) return; + this.#operator = { ...DEFAULT_FULL_TEXT_TELEMETRY_POLICY }; + this.#consumers.clear(); + this.#updatePolicy(); + this.#stopped = true; + this.#pending.clear(); + this.#entries.clear(); + this.#registered = new WeakMap(); + for (const group of this.#groups.values()) group.entries.clear(); + this.#groups.clear(); + } + + #updatePolicy() { + clearTimeout(this.#timer); + this.#timer = undefined; + this.#revision++; + const demands = [...this.#consumers].map((consumer) => consumer.policy); + if (this.#operator.enabled) demands.push(this.#operator); + this.#policy = { + enabled: demands.length > 0, + intervalMilliseconds: Math.min(...demands.map((policy) => policy.intervalMilliseconds), 2_147_483_647), + histograms: demands.some((policy) => policy.histograms), + snapshotBytes: demands.some((policy) => policy.snapshotBytes), + }; + let offset = 0; + const step = this.#policy.intervalMilliseconds / Math.max(1, this.#entries.size); + for (const entry of this.#entries) { + entry.configured = false; + if (!entry.closing) this.#configure(entry); + entry.nextAt = performance.now() + step * ++offset; + } + this.#prune(); + this.#schedule(); + } + + #configure(entry: Entry): boolean { + try { + entry.handle.configureTelemetry({ + histograms: this.#policy.histograms, + snapshotBytes: this.#policy.snapshotBytes, + }); + entry.configured = true; + return true; + } catch { + this.#failed(entry, 'configuration-error'); + return false; + } + } + + #failed(entry: Entry, error: FullTextTelemetrySample['error']) { + if (entry.closing) entry.group.complete = false; + if (!entry.warned) { + entry.warned = true; + try { + this.#onError(); + } catch {} + } + this.#publish(entry, undefined, error); + } + + #observe(entry: Entry) { + if (!this.collecting || !this.#entries.has(entry)) return; + if (!entry.closing && !entry.configured && !this.#configure(entry)) return; + try { + const snapshot = entry.handle.telemetry(); + const flat = flattenFullTextTelemetry(snapshot); + for (const [name, value] of flat.counters) { + const previous = entry.previous.get(name) ?? 0n; + const delta = name.startsWith('native_') ? fullTextNativeCounterDelta(value, previous) : value - previous; + if (delta < 0n) entry.group.complete = false; + else entry.group.totals.set(name, (entry.group.totals.get(name) ?? 0n) + delta); + entry.previous.set(name, value); + } + entry.flat = flat; + entry.observedAt = performance.now(); + entry.warned = false; + if (!snapshot.available && entry.closing) entry.group.complete = false; + this.#publish(entry, snapshot); + } catch { + this.#failed(entry, 'collection-error'); + } + } + + #publish(entry: Entry, snapshot?: FullTextTelemetry, error?: FullTextTelemetrySample['error'], stage?: 'closing') { + const sampledAt = Date.now(); + const revision = this.#revision; + if (this.#operator.enabled) { + let newest = entry; + for (const candidate of entry.group.entries) + if (!candidate.closing && candidate.flat && (newest.closing || candidate.observedAt > newest.observedAt)) + newest = candidate; + const row: FullTextTelemetryRow = { + ...newest.flat?.gauges, + metric: 'fulltext', + path: entry.group.index, + method: entry.group.role, + perThread: true, + cumulative: true, + series: entry.group.series, + sampledAt, + complete: entry.group.complete, + collectionError: error ?? null, + droppedRows: this.#droppedRows.toString(), + observationAgeMilliseconds: entry.observedAt ? performance.now() - entry.observedAt : null, + activeHandles: [...entry.group.entries].filter((candidate) => !candidate.closing).length, + closingHandles: [...entry.group.entries].filter((candidate) => candidate.closing).length, + available: !error && snapshot?.available === true, + }; + for (const [name, value] of entry.group.totals) row[name] = value.toString(); + for (const name of Object.keys(row)) { + if ( + /^native_operations_[A-Za-z]+_dispatched$/.test(name) || + /^native_queues_[A-Za-z]+_(commands|bytes|maxCommands|maxBytes|oldestAgeNanoseconds)$/.test(name) + ) { + let total = 0n; + let observed = false; + for (const candidate of entry.group.entries) { + const value = candidate.flat?.gauges[name]; + if (!candidate.closing && typeof value === 'string' && /^\d+$/.test(value)) { + const number = BigInt(value); + total = name.endsWith('oldestAgeNanoseconds') ? (number > total ? number : total) : total + number; + observed = true; + } + } + row[name] = observed ? total.toString() : null; + } + } + if (!this.#pending.has(entry.group) && this.#pending.size >= this.#maxPendingRows) { + const oldest = this.#pending.keys().next().value!; + this.#pending.delete(oldest); + oldest.complete = false; + this.#droppedRows++; + row.droppedRows = this.#droppedRows.toString(); + this.#prune(); + } + this.#pending.set(entry.group, row); + try { + this.#onPending(); + } catch {} + } + for (const consumer of [...this.#consumers]) { + if (revision !== this.#revision) break; + if (!this.#consumers.has(consumer) || consumer.busy) continue; + try { + const result = consumer.callback({ + index: entry.group.index, + role: entry.group.role, + series: entry.group.series, + sampledAt, + snapshot, + error, + stage, + }); + if (result && typeof result.then === 'function') { + consumer.busy = true; + void result + .catch(() => { + try { + this.#onError(); + } catch {} + }) + .finally(() => { + consumer.busy = false; + }); + } + } catch { + try { + this.#onError(); + } catch {} + } + } + } + + #schedule() { + clearTimeout(this.#timer); + this.#timer = undefined; + if (!this.collecting) return; + let next: Entry | undefined; + for (const entry of this.#entries) if (!entry.closing && (!next || entry.nextAt < next.nextAt)) next = entry; + if (!next) return; + const entry = next; + this.#timer = setTimeout( + () => { + this.#timer = undefined; + entry.nextAt = performance.now() + this.#policy.intervalMilliseconds; + this.#observe(entry); + this.#schedule(); + }, + Math.max(0, next.nextAt - performance.now()) + ).unref(); + } + + #prune() { + for (const [key, group] of this.#groups) + if (!group.entries.size && !this.#pending.has(group)) this.#groups.delete(key); + } +} diff --git a/resources/analytics/fullTextTelemetryRuntime.ts b/resources/analytics/fullTextTelemetryRuntime.ts new file mode 100644 index 0000000000..3caee64ea7 --- /dev/null +++ b/resources/analytics/fullTextTelemetryRuntime.ts @@ -0,0 +1,90 @@ +import { createHash } from 'node:crypto'; +import { onStartup } from '../../utility/lifecycle.ts'; +import { registerShutdownDrain } from '../../components/shutdownDrain.ts'; +import { loggerWithTag } from '../../utility/logging/logger.ts'; +import { watchFullTextTelemetryConfiguration } from '../../config/fullTextTelemetryConfiguration.ts'; +import { addAnalyticsListener, requestAnalyticsFlush, checkAnalyticsEnabled } from './write.ts'; +import { + FullTextTelemetryCollector, + resolveFullTextTelemetryPolicy, + type FullTextTelemetryHandle, + type FullTextTelemetrySample, + type FullTextTelemetryPolicy, +} from './fullTextTelemetry.ts'; + +const logger = loggerWithTag('fulltext-analytics'); +let operatorPolicy = resolveFullTextTelemetryPolicy(undefined); +let started = false; +let stopWatching: (() => void) | undefined; +let removeListener: (() => void) | undefined; +let enabledSince = 0; +let persistenceActive = false; +let nextWarningAt = 0; + +export const fullTextTelemetryCollector = new FullTextTelemetryCollector({ + onPending: () => { + if (operatorPolicy.enabled && checkAnalyticsEnabled()) requestAnalyticsFlush(); + }, + onError: () => { + if (Date.now() < nextWarningAt) return; + nextWarningAt = Date.now() + 60_000; + logger.warn?.('A full-text telemetry observation or consumer failed'); + }, +}); + +export function setFullTextAnalyticsEnabled(enabled: boolean): void { + const next = operatorPolicy.enabled && enabled; + if (next && !persistenceActive) enabledSince = Date.now(); + persistenceActive = next; + fullTextTelemetryCollector.setOperatorPolicy({ ...operatorPolicy, enabled: next }); +} + +export function acceptsFullTextTelemetryRow(row: Record): boolean { + return ( + persistenceActive && checkAnalyticsEnabled() && typeof row.sampledAt === 'number' && row.sampledAt >= enabledSince + ); +} + +export function initializeFullTextTelemetry(): void { + if (started) return; + started = true; + removeListener = addAnalyticsListener((metrics) => { + const rows = fullTextTelemetryCollector.takeRows(); + if (operatorPolicy.enabled && checkAnalyticsEnabled()) metrics.push(...rows); + }); + stopWatching = watchFullTextTelemetryConfiguration((value) => { + const next = resolveFullTextTelemetryPolicy(value); + operatorPolicy = next; + setFullTextAnalyticsEnabled(checkAnalyticsEnabled()); + }); +} + +export function subscribeFullTextTelemetry( + callback: (sample: FullTextTelemetrySample) => void | Promise, + options?: Omit, 'enabled'> +): () => void { + initializeFullTextTelemetry(); + return fullTextTelemetryCollector.subscribe(callback, options); +} + +export function registerNativeFullTextTelemetry(handle: unknown, index: string, role: 'writer' | 'reader'): void { + try { + initializeFullTextTelemetry(); + const native = handle as FullTextTelemetryHandle; + if (typeof native?.telemetry !== 'function' || typeof native.configureTelemetry !== 'function') return; + const label = Buffer.byteLength(index) <= 256 ? index : `index:${createHash('sha256').update(index).digest('hex')}`; + fullTextTelemetryCollector.register(native, label, role); + } catch { + logger.warn?.('Could not register a full-text telemetry handle'); + } +} + +onStartup(initializeFullTextTelemetry); +registerShutdownDrain({ + hasWork: () => false, + drain: async () => { + stopWatching?.(); + removeListener?.(); + fullTextTelemetryCollector.stop(); + }, +}); diff --git a/resources/analytics/metadata.ts b/resources/analytics/metadata.ts index 127627c889..0abd1e2f11 100644 --- a/resources/analytics/metadata.ts +++ b/resources/analytics/metadata.ts @@ -1,4 +1,5 @@ export const METRIC = { + FULL_TEXT: 'fulltext', TABLE_SIZE: 'table-size', DATABASE_SIZE: 'database-size', STORAGE_VOLUME: 'storage-volume', diff --git a/resources/analytics/write.ts b/resources/analytics/write.ts index eceaa6bcfb..bb224c61a8 100644 --- a/resources/analytics/write.ts +++ b/resources/analytics/write.ts @@ -15,6 +15,7 @@ import { server } from '../../server/Server.ts'; import * as fs from 'node:fs'; import { getAnalyticsHostnameTable, nodeIds, stableNodeId } from './hostnames.ts'; import { METRIC } from './metadata.ts'; +import { acceptsFullTextTelemetryRow, setFullTextAnalyticsEnabled } from './fullTextTelemetryRuntime.ts'; import { readEventLoopDelay, resetEventLoopDelay, @@ -52,6 +53,7 @@ let analyticsGeneration = 0; onStartup(() => { analyticsEnabled = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) > -1; analyticsReadOnlyChecked = false; + setFullTextAnalyticsEnabled(checkAnalyticsEnabled()); }); // Check read-only mode lazily to avoid circular dependency at module load time. The event loop @@ -68,6 +70,7 @@ export function checkAnalyticsEnabled(): boolean { /** Resets this thread's analytics window; other threads keep their own. */ export function setAnalyticsEnabled(enabled: boolean) { analyticsEnabled = enabled; + setFullTextAnalyticsEnabled(enabled); if (enabled) startEventLoopDelayMonitor(); else stopEventLoopDelayMonitor(); clearTimeout(sendAnalyticsTimeout); // reset this @@ -181,6 +184,14 @@ const analyticsAggregateListeners = []; export function addAnalyticsListener(callback) { analyticsListeners.push(callback); + return () => { + const index = analyticsListeners.indexOf(callback); + if (index >= 0) analyticsListeners.splice(index, 1); + }; +} + +export function requestAnalyticsFlush(): void { + if (checkAnalyticsEnabled() && !sendAnalyticsTimeout) contextStorage.exit(sendAnalytics); } const IDEAL_PERCENTILES = [0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99, 0.999, 1]; @@ -305,7 +316,7 @@ async function flushAnalytics() { depth: queueDepths.readDepth, maxDepth: queueDepths.readMaxDepth, }); - for (const listener of analyticsListeners) { + for (const listener of [...analyticsListeners]) { if (generation !== analyticsGeneration) return; listener(metrics); } @@ -1070,6 +1081,15 @@ async function aggregation(fromPeriod, toPeriod = 60000) { // key, whose action name a request body chooses, always ends in `-operation`. if (perThread) key += '\0' + threadId; let action = aggregateActions.get(key); + if (entry.cumulative) { + if (!action || entry.sampledAt >= action.sampledAt) { + const latest = { ...entry, period: toPeriod, thread: threadId }; + delete latest.perThread; + delete latest.threadId; + aggregateActions.set(key, latest); + } + return; + } if (action) { const byThread = Boolean(action.byThread); if (byThread) { @@ -1363,6 +1383,9 @@ function recordAnalytics(message, worker?) { // Skip writes in read-only mode if (isReadOnlyMode()) return; const report = message.report; + report.metrics = report.metrics.filter( + (metric) => metric.metric !== METRIC.FULL_TEXT || acceptsFullTextTelemetryRow(metric) + ); report.threadId = worker?.threadId || threadId; // Add system information stats as well for (const metric of report.metrics) { diff --git a/resources/derivedIndexes.ts b/resources/derivedIndexes.ts index bbc0b2700b..c8e9c71c86 100644 --- a/resources/derivedIndexes.ts +++ b/resources/derivedIndexes.ts @@ -419,6 +419,7 @@ export function attachDerivedIndexes( indexId: id, storePath: Table.primaryStore.rootStore.path, storeName: `${Table.tableName}/${definition.name}`, + telemetryLabel: `${Table.databaseName}.${Table.tableName}/${definition.name}`, sourceGeneration: `${Table.tableId}:${generation}`, limits: { ...FULL_TEXT_READER_LIMITS }, ...(fullTextTest ? { binding: fullTextTest.binding } : null), @@ -478,6 +479,7 @@ export function attachDerivedIndexes( storePath: Table.primaryStore.rootStore.path, storeName, sourceGeneration: `${Table.tableId}:${generation}`, + telemetryLabel: `${Table.databaseName}.${Table.tableName}/${definition.name}`, fields: definition.fields.map(({ name, weight }) => ({ name, weight })), analyzer: storage.analyzer, stopWords: storage.stopWords, diff --git a/resources/indexes/DESIGN.md b/resources/indexes/DESIGN.md index dcf8b9ad40..5a18f2f957 100644 --- a/resources/indexes/DESIGN.md +++ b/resources/indexes/DESIGN.md @@ -562,7 +562,7 @@ When more hits are needed, switch to `searchPage` on the same reader. Ordinary s Byte-aware batch sizes adapt to filter yield. Oversized responses retry the unchanged boundary at a smaller size under the same deadline. Exhausting the snapshot returns remaining eligible rows. `maxRawHitsPerRequest` caps fetched hits including bounded replay; `maxResultsPerRequest` independently caps retained offset plus limit. Both default to 10,000. Before replay, the first page total supplies a lower bound on the new hits needed to fill or exhaust the request; reject immediately if even that work plus replay cannot fit. Use the smaller of the remaining target and remaining native total so a shorter exhausted result remains valid. A lower-bound total equal to the first batch does not prove that additional hits exist, so only replay can be reserved; traversal must distinguish actual exhaustion from more matches before rejecting. Hitting a bound before exhaustion or filling the requested page throws an explicit error. These are not bounds on Tantivy scorer visits or uninterrupted query construction. Unbounded queries that reach a result/work ceiling must use a limit; they never silently return the retained prefix. -Harper requires Fulltext 0.8.0, native ABI 13 and Query API 7. Traversal response capacity reserves 93 bytes before hits plus 13 bytes per versioned hit before ID/version strings; advertised page and response limits determine the batch ceiling. +Harper requires Fulltext 0.9.0, native ABI 14, query API 8 and telemetry API 1. Traversal response capacity reserves 93 bytes before hits plus 13 bytes per versioned hit before ID/version strings; advertised page and response limits determine the batch ceiling. Read request policy from its three registered leaf parameters and runtime policy from `storage_fullText_runtime`. The legacy flattener does not retain camel-case aggregate containers such as `storage_fullText`; using that aggregate silently selects defaults. HTTP request settings follow normal worker-restart semantics; native runtime settings remain captured for the full process. Generated configuration omits `storage.fullText.query`; the resolver supplies defaults and preserves partial overrides. `storage.fullText.query` uses that same snapshot for reader term/clause policy and the fuzzy preview maximum; writer limits and persisted identity stay unchanged. Retrieval and highlighting retain the same fuzzy leaf controls. diff --git a/resources/indexes/fullTextDerivedIndex.ts b/resources/indexes/fullTextDerivedIndex.ts index 6a2519de51..73115ba8fe 100644 --- a/resources/indexes/fullTextDerivedIndex.ts +++ b/resources/indexes/fullTextDerivedIndex.ts @@ -17,6 +17,7 @@ import type { DerivedIndexPositions, } from '../derivedIndexRuntime.ts'; import { toBufferKey } from 'ordered-binary'; +import { fullTextTelemetryCollector } from '../analytics/fullTextTelemetryRuntime.ts'; import { loggerWithTag } from '../../utility/logging/logger.ts'; const logger = loggerWithTag('fulltext-derived-index'); @@ -791,7 +792,13 @@ export class FullTextDerivedIndexBackend implements DerivedIndexBackend { ): Promise { let closeOperation = this.#closeOperations.get(engine); if (!closeOperation) { - closeOperation = engine.close(options); + fullTextTelemetryCollector.beforeClose(engine); + try { + closeOperation = engine.close(options).finally(() => fullTextTelemetryCollector.afterClose(engine)); + } catch (error) { + fullTextTelemetryCollector.afterClose(engine); + throw error; + } this.#closeOperations.set(engine, closeOperation); const installed = closeOperation; void installed.catch(() => { diff --git a/resources/indexes/fullTextNativeBinding.ts b/resources/indexes/fullTextNativeBinding.ts index feeae7253d..bd086bba12 100644 --- a/resources/indexes/fullTextNativeBinding.ts +++ b/resources/indexes/fullTextNativeBinding.ts @@ -1,3 +1,4 @@ +import type { FullTextTelemetry, ConfigureTelemetryOptions } from '@harperfast/fulltext/native'; import type { FullTextDerivedIndexEngine } from './fullTextDerivedIndex.ts'; import * as env from '../../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS } from '../../utility/hdbTerms.ts'; @@ -9,10 +10,11 @@ import { import { getFullTextRuntimeConfiguration, FullTextWorkerInitializationError } from './fullTextRuntimeConfiguration.ts'; import { loggerWithTag } from '../../utility/logging/logger.ts'; -const FULLTEXT_NATIVE_ABI_VERSION = 13; +const FULLTEXT_NATIVE_ABI_VERSION = 14; const FULLTEXT_LIFECYCLE_API_VERSION = 1; const FULLTEXT_MUTATION_BATCH_API_VERSION = 5; -const FULLTEXT_QUERY_API_VERSION = 7; +const FULLTEXT_QUERY_API_VERSION = 8; +const FULLTEXT_TELEMETRY_API_VERSION = 1; const logger = loggerWithTag('fulltext-derived-index'); export interface NativeFullTextRuntimeBudget { @@ -95,7 +97,12 @@ export type NativeFullTextTraceResult = { }>; }; -export interface NativeFullTextReader { +export interface NativeFullTextTelemetryHandle { + telemetry(): FullTextTelemetry; + configureTelemetry(options: ConfigureTelemetryOptions): void; +} + +export interface NativeFullTextReader extends NativeFullTextTelemetryHandle { readonly committedPayload?: string; reload(): Promise; search( @@ -127,6 +134,7 @@ export type NativeFullTextRuntimeInfo = { lifecycleApiVersion: number; mutationBatchApiVersion: number; queryApiVersion: number; + telemetryApiVersion: number; storageBackends: ReadonlyArray<'native'>; limits: { maxConfiguredQueryTerms: number; @@ -170,7 +178,11 @@ export interface NativeFullTextModule { configureNativeFullTextRuntime(limits: NativeFullTextRuntimeBudget): void; NativeFullTextIndex: { prototype: Pick & - Pick; + Pick & + NativeFullTextTelemetryHandle; + }; + NativeFullTextReader: { + prototype: NativeFullTextTelemetryHandle; }; runtimeInfo(): Promise; validateNativeFullTextIndexOptions( @@ -264,10 +276,14 @@ export async function validateFullTextNativeBinding(module: unknown): Promise>; + prototype?: Partial< + FullTextDerivedIndexEngine & Pick & NativeFullTextTelemetryHandle + >; }; } )?.NativeFullTextIndex?.prototype; + const readerPrototype = (module as { NativeFullTextReader?: { prototype?: Partial } }) + ?.NativeFullTextReader?.prototype; if ( !module || typeof module !== 'object' || @@ -279,6 +295,12 @@ export async function validateFullTextNativeBinding(module: unknown): Promise Promise); }; @@ -969,6 +971,7 @@ export class FullTextQueryIndex { reloadFailures: 0, }; this.#readerSlot = slot; + registerNativeFullTextTelemetry(reader, this.#options.telemetryLabel ?? this.#options.indexId, 'reader'); return slot; } catch (error) { if (canReload && error instanceof FullTextReaderPublicationError) { @@ -1055,7 +1058,11 @@ export class FullTextQueryIndex { #closeReaderSlot(slot: ReaderSlot): Promise { if (!slot.closeOperation) { const operation = Promise.resolve() - .then(() => slot.reader.close()) + .then(() => { + fullTextTelemetryCollector.beforeClose(slot.reader); + return slot.reader.close(); + }) + .finally(() => fullTextTelemetryCollector.afterClose(slot.reader)) .then((result) => { this.#retiredReaderSlots.delete(slot); if (result?.cleanupError) this.#warnReaderClose(result.cleanupError); diff --git a/resources/indexes/nativeFullTextDerivedIndexLifecycle.ts b/resources/indexes/nativeFullTextDerivedIndexLifecycle.ts index 2ef77e1b10..11aed031f1 100644 --- a/resources/indexes/nativeFullTextDerivedIndexLifecycle.ts +++ b/resources/indexes/nativeFullTextDerivedIndexLifecycle.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'; import { existsSync } from 'node:fs'; import { dirname, isAbsolute, join, resolve } from 'node:path'; import { loggerWithTag } from '../../utility/logging/logger.ts'; +import { registerNativeFullTextTelemetry } from '../analytics/fullTextTelemetryRuntime.ts'; import { FullTextDerivedIndexBackend, HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES, @@ -24,6 +25,7 @@ export type NativeFullTextDerivedIndexLifecycleOptions = NativeFullTextIndexConf storeName: string; indexId: string; sourceGeneration: string; + telemetryLabel?: string; binding?: NativeFullTextModule | (() => Promise); beforeReset?: (ownerEpoch: bigint) => Promise; afterReset?: (ownerEpoch: bigint) => void | Promise; @@ -91,10 +93,12 @@ export class NativeFullTextDerivedIndexLifecycle { } async open(): Promise { - return this.#requireBinding().openNativeFullTextIndex({ + const engine = await this.#requireBinding().openNativeFullTextIndex({ ...this.#nativeOptions(), limits: this.#options.limits, }); + registerNativeFullTextTelemetry(engine, this.#options.telemetryLabel ?? this.#options.indexId, 'writer'); + return engine; } async reset(ownerEpoch: bigint): Promise { diff --git a/unitTests/config/fullTextTelemetryConfiguration.test.js b/unitTests/config/fullTextTelemetryConfiguration.test.js new file mode 100644 index 0000000000..511bcdb5dc --- /dev/null +++ b/unitTests/config/fullTextTelemetryConfiguration.test.js @@ -0,0 +1,130 @@ +const assert = require('node:assert'); +const { mkdtempSync, writeFileSync, rmSync } = require('node:fs'); +const { join } = require('node:path'); +const { stringify } = require('yaml'); +const { RootConfigWatcher } = require('#src/config/RootConfigWatcher'); +const { watchFullTextTelemetryConfiguration } = require('#src/config/fullTextTelemetryConfiguration'); +const { resolveFullTextTelemetryPolicy } = require('#src/resources/analytics/fullTextTelemetry'); +const env = require('#src/utility/environment/environmentManager'); +const { updateConfigObject } = require('#src/config/configUtils'); +const { setupTestDBPath } = require('../testUtils'); +const { waitFor } = require('../waitFor'); + +const PARAM = 'analytics_fullText'; +const root = (fullText) => ({ analytics: { fullText } }); + +// Every observer here uses a real file and shared-watcher implementation; no native binding is loaded. +describe('Live full-text telemetry configuration', function () { + this.timeout(20_000); + let directory, file, watcher, stop, saved, savedOverrides, savedEnv; + beforeEach(() => { + saved = structuredClone(env.get(PARAM)); + savedOverrides = Object.entries(env.getConfigOverrides() ?? {}).filter( + ([key]) => key === PARAM || key.startsWith(PARAM + '_') + ); + savedEnv = Object.fromEntries( + ['HARPER_CONFIG', 'HARPER_DEFAULT_CONFIG', 'HARPER_SET_CONFIG'].map((key) => [key, process.env[key]]) + ); + for (const key of Object.keys(savedEnv)) delete process.env[key]; + directory = mkdtempSync(join(setupTestDBPath(), 'telemetry-config-')); + file = join(directory, 'config.yaml'); + }); + afterEach(() => { + stop?.(); + watcher?.close(); + stop = watcher = undefined; + for (const [key, value] of Object.entries(savedEnv)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + env.refreshConfigSubtree(PARAM, root({ enabled: Symbol('previous') }), {}, resolveFullTextTelemetryPolicy); + updateConfigObject(PARAM, saved); + for (const [name, value] of savedOverrides) env.setProperty(name, value); + rmSync(directory, { recursive: true, force: true }); + }); + function observe(value, samples) { + writeFileSync(file, stringify(root(value))); + watcher = new RootConfigWatcher(file); + stop = watchFullTextTelemetryConfiguration((policy) => samples.push(policy), watcher); + } + it('enables, retains a valid policy after an invalid edit, disables and unsubscribes', async () => { + const samples = []; + observe({ enabled: true, intervalMilliseconds: 1000 }, samples); + await waitFor(() => samples.at(-1)?.enabled === true); + const valid = samples.at(-1); + const unchangedCount = samples.length; + writeFileSync(file, stringify({ ...root({ enabled: true, intervalMilliseconds: 1000 }), unrelated: 1 })); + await waitFor(() => watcher.config?.unrelated === 1); + assert.strictEqual(samples.length, unchangedCount, 'unrelated config changes must not reset collection cadence'); + writeFileSync(file, stringify(root({ enabled: 'invalid' }))); + await waitFor(() => watcher.config?.analytics.fullText.enabled === 'invalid'); + assert.deepStrictEqual(samples.at(-1), valid); + writeFileSync(file, stringify(root({ enabled: false }))); + await waitFor(() => samples.at(-1)?.enabled === false); + stop(); + const count = samples.length; + writeFileSync(file, stringify(root({ enabled: true }))); + await waitFor(() => watcher.config?.analytics.fullText.enabled === true); + assert.strictEqual(samples.length, count); + }); + it('disables on file deletion and resumes on recreation', async () => { + const samples = []; + observe({ enabled: true }, samples); + await waitFor(() => samples.at(-1)?.enabled === true); + rmSync(file); + await waitFor(() => samples.at(-1)?.enabled === false); + writeFileSync(file, stringify(root({ enabled: true }))); + await waitFor(() => samples.at(-1)?.enabled === true); + }); + it('applies a recreated changed file once and ignores invalid revival policy', async () => { + const samples = []; + observe({ enabled: true }, samples); + await waitFor(() => samples.at(-1)?.enabled === true); + rmSync(file); + await waitFor(() => samples.at(-1)?.enabled === false); + const disabledCount = samples.length; + writeFileSync(file, stringify(root({ enabled: 'invalid' }))); + await waitFor(() => watcher.config?.analytics.fullText.enabled === 'invalid'); + assert.strictEqual(samples.length, disabledCount); + writeFileSync(file, stringify(root({ enabled: true, intervalMilliseconds: 2000 }))); + await waitFor(() => samples.at(-1)?.enabled === true); + assert.strictEqual(samples.length, disabledCount + 1); + assert.strictEqual(samples.at(-1).intervalMilliseconds, 2000); + }); + it('honors reasserted environment policy over edits to the file', async () => { + process.env.HARPER_SET_CONFIG = JSON.stringify(root({ enabled: true, intervalMilliseconds: 2000 })); + const samples = []; + observe({ enabled: false }, samples); + await watcher.ready; + await waitFor(() => samples.at(-1)?.enabled === true); + writeFileSync(file, stringify(root({ enabled: false, intervalMilliseconds: 3000 }))); + await waitFor(() => watcher.config?.analytics.fullText.intervalMilliseconds === 3000); + assert.strictEqual(samples.at(-1).intervalMilliseconds, 2000); + }); + for (const override of [ + [PARAM, { enabled: true, intervalMilliseconds: 2000 }], + [PARAM + '_enabled', true], + ]) + it(`preserves unchanged ${override[0]} override and retires it after its own file edit`, () => { + const before = root({ enabled: false, intervalMilliseconds: 1000 }); + env.setProperty(...override); + assert.strictEqual( + env.refreshConfigSubtree(PARAM, undefined, before, resolveFullTextTelemetryPolicy).enabled, + true + ); + const unrelated = { ...before, unrelated: 1 }; + assert.strictEqual( + env.refreshConfigSubtree(PARAM, before, unrelated, resolveFullTextTelemetryPolicy).enabled, + true + ); + assert.throws(() => + env.refreshConfigSubtree(PARAM, before, root({ enabled: 'invalid' }), resolveFullTextTelemetryPolicy) + ); + assert.deepStrictEqual(env.getConfigOverrides()[override[0]], override[1]); + assert.strictEqual( + env.refreshConfigSubtree(PARAM, before, { analytics: {} }, resolveFullTextTelemetryPolicy).enabled, + false + ); + assert.strictEqual(env.getConfigOverrides()?.[override[0]], undefined); + }); +}); diff --git a/unitTests/resources/analytics/aggregationCycle.test.js b/unitTests/resources/analytics/aggregationCycle.test.js index bf9e57d3ff..173662fff9 100644 --- a/unitTests/resources/analytics/aggregationCycle.test.js +++ b/unitTests/resources/analytics/aggregationCycle.test.js @@ -179,6 +179,54 @@ describe('analytics aggregation cycle', () => { analytics.setAnalyticsEnabled(false); }); + it('keeps the latest exact cumulative sample per worker and logical index without averaging counters', async function () { + this.timeout(30000); + await drainLiveReports(); + const first = lastRawKey() + 1; + const metric = 'fulltext'; + const entry = (sampledAt, value, extra = {}) => ({ + metric, + path: 'AggWindowFulltext', + method: 'reader', + perThread: true, + cumulative: true, + series: 'worker-one-series', + sampledAt, + completed: value, + available: true, + pending: false, + error: null, + ...extra, + }); + await seedRawReports([ + sampleReport(first, 1, entry(10, '9007199254740993')), + sampleReport(first + 1, 1, entry(20, '9007199254740997', { pending: true, error: 'E_STORAGE' })), + sampleReport(first + 2, 1, entry(15, '9007199254740995')), + sampleReport(first + 3, 2, entry(30, '7', { series: 'worker-two-series' })), + ]); + await nextPeriod(); + await runCycle(); + const fullTextRows = () => aggregatedMetrics(metric, first + 3).filter((row) => row.path === 'AggWindowFulltext'); + await waitFor(async () => { + await runCycle(); + return fullTextRows().length === 2; + }, 10000); + const rows = fullTextRows().sort((a, b) => a.thread - b.thread); + assert.strictEqual(rows[0].completed, '9007199254740997'); + assert.strictEqual(rows[0].sampledAt, 20); + assert.strictEqual(rows[0].pending, true); + assert.strictEqual(rows[0].error, 'E_STORAGE'); + assert.strictEqual(rows[1].completed, '7'); + assert.strictEqual(rows[0].mean, undefined); + assert.strictEqual(rows[0].ratio, undefined); + const reader = require('#src/resources/analytics/read'); + await waitFor(async () => { + for await (const value of await reader.get(metric, { startTime: first, endTime: Date.now() + 1000 })) + if (value.completed === '9007199254740997') return true; + return false; + }, 10000); + }); + it('keeps its place when a cycle finds nothing to aggregate', async function () { this.timeout(30000); const bootstrapKey = lastRawKey(); diff --git a/unitTests/resources/derivedIndexRuntimeRocks.test.js b/unitTests/resources/derivedIndexRuntimeRocks.test.js index 451181ae49..44289d8887 100644 --- a/unitTests/resources/derivedIndexRuntimeRocks.test.js +++ b/unitTests/resources/derivedIndexRuntimeRocks.test.js @@ -795,6 +795,12 @@ class FakeNativeFullTextModule { publish() {} close() {} searchPage() {} + telemetry() {} + configureTelemetry() {} + }; + this.NativeFullTextReader = class { + telemetry() {} + configureTelemetry() {} }; this.states = new Map(); this.opens = []; @@ -805,11 +811,12 @@ class FakeNativeFullTextModule { return { packageVersion: 'test', tantivyVersion: 'test', - nativeAbiVersion: 13, + nativeAbiVersion: 14, queryClassIsolationMinimumSearchThreads: 2, lifecycleApiVersion: 1, mutationBatchApiVersion: 5, - queryApiVersion: 7, + queryApiVersion: 8, + telemetryApiVersion: 1, storageBackends: ['native'], limits: { maxCommitPayloadBytes: 64 * 1024, diff --git a/unitTests/resources/fixtures/fullTextUpgradeProcess.cjs b/unitTests/resources/fixtures/fullTextUpgradeProcess.cjs index a89a95a591..69a04de413 100644 --- a/unitTests/resources/fixtures/fullTextUpgradeProcess.cjs +++ b/unitTests/resources/fixtures/fullTextUpgradeProcess.cjs @@ -6,13 +6,17 @@ const assert = require('node:assert'); const chunks = []; for await (const chunk of process.stdin) chunks.push(chunk); const { mode, options, documents, payload, version = '0.5.0' } = JSON.parse(Buffer.concat(chunks)); - assert(['0.5.0', '0.7.0'].includes(version)); + assert(['0.5.0', '0.7.0', '0.8.0'].includes(version)); const binding = await import( - version === '0.7.0' ? '@harperfast/fulltext-v07/native' : '@harperfast/fulltext-previous/native' + version === '0.8.0' + ? '@harperfast/fulltext-v08/native' + : version === '0.7.0' + ? '@harperfast/fulltext-v07/native' + : '@harperfast/fulltext-previous/native' ); const info = await binding.runtimeInfo(); assert.strictEqual(info.packageVersion, version); - assert.strictEqual(info.tantivyVersion, version === '0.7.0' ? '0.27.0' : '0.26.2'); + assert.strictEqual(info.tantivyVersion, version !== '0.5.0' ? '0.27.0' : '0.26.2'); let index; let ghostIds; try { @@ -39,7 +43,7 @@ const assert = require('node:assert'); try { const reader = await binding.openNativeFullTextReader(options); try { - if (version === '0.7.0') searchIds = (await reader.search({ text: 'catalog' })).hits.map(({ id }) => id); + if (version !== '0.5.0') searchIds = (await reader.search({ text: 'catalog' })).hits.map(({ id }) => id); } finally { await reader.close(); } diff --git a/unitTests/resources/fullTextNativeTestBinding.js b/unitTests/resources/fullTextNativeTestBinding.js index abbd87c2ce..6046c20e93 100644 --- a/unitTests/resources/fullTextNativeTestBinding.js +++ b/unitTests/resources/fullTextNativeTestBinding.js @@ -23,6 +23,12 @@ class FullTextNativeTestBinding { publish() {} close() {} searchPage() {} + telemetry() {} + configureTelemetry() {} + }; + this.NativeFullTextReader = class { + telemetry() {} + configureTelemetry() {} }; } @@ -30,11 +36,12 @@ class FullTextNativeTestBinding { return { packageVersion: 'test', tantivyVersion: 'test', - nativeAbiVersion: 13, + nativeAbiVersion: 14, queryClassIsolationMinimumSearchThreads: 2, lifecycleApiVersion: 1, mutationBatchApiVersion: 5, - queryApiVersion: 7, + queryApiVersion: 8, + telemetryApiVersion: 1, storageBackends: ['native'], limits: { maxCommitPayloadBytes: 64 * 1024, diff --git a/unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js b/unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js index 76c5fa9215..6d83ccfeec 100644 --- a/unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js +++ b/unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js @@ -11,7 +11,10 @@ const { NativeFullTextDerivedIndexLifecycle, retireNativeFullTextDerivedIndexStorage, } = require('#src/resources/indexes/nativeFullTextDerivedIndexLifecycle'); -const { loadFullTextNativeBinding } = require('#src/resources/indexes/fullTextNativeBinding'); +const { + loadFullTextNativeBinding, + validateFullTextNativeBinding, +} = require('#src/resources/indexes/fullTextNativeBinding'); const { DERIVED_INDEX_ACCEPTED, DERIVED_INDEX_DEFERRED, @@ -60,6 +63,12 @@ class FakeNativeModule { publish() {} close() {} searchPage() {} + telemetry() {} + configureTelemetry() {} + }; + this.NativeFullTextReader = class { + telemetry() {} + configureTelemetry() {} }; this.runtimeInfoCalls = 0; this.inspection = { state: 'missing' }; @@ -76,11 +85,12 @@ class FakeNativeModule { return { packageVersion: 'test', tantivyVersion: 'test', - nativeAbiVersion: 13, + nativeAbiVersion: 14, queryClassIsolationMinimumSearchThreads: 2, lifecycleApiVersion: 1, mutationBatchApiVersion: 5, - queryApiVersion: 7, + queryApiVersion: 8, + telemetryApiVersion: 1, storageBackends: ['native'], limits: { maxCommitPayloadBytes: this.maxCommitPayloadBytes, @@ -156,7 +166,7 @@ describe('NativeFullTextDerivedIndexLifecycle', () => { fs.mkdirSync(storePath, { recursive: true }); }); - it('fits a v7 traversal page of maximum-size native identities in the advertised response envelope', async function () { + it('fits a traversal page of maximum-size native identities in the advertised response envelope', async function () { this.timeout(60000); if (!(await supportsPublishedFullTextBinding())) this.skip(); const binding = await loadFullTextNativeBinding(); @@ -205,10 +215,11 @@ describe('NativeFullTextDerivedIndexLifecycle', () => { const runtime = await binding.runtimeInfo(); assert.strictEqual(runtime.packageVersion, fullTextPackageVersion); assert.strictEqual(runtime.tantivyVersion, '0.27.0'); - assert.strictEqual(runtime.nativeAbiVersion, 13); + assert.strictEqual(runtime.nativeAbiVersion, 14); assert.strictEqual(runtime.lifecycleApiVersion, 1); assert.strictEqual(runtime.mutationBatchApiVersion, 5); - assert.strictEqual(runtime.queryApiVersion, 7); + assert.strictEqual(runtime.queryApiVersion, 8); + assert.strictEqual(runtime.telemetryApiVersion, 1); assert.strictEqual(runtime.queryClassIsolationMinimumSearchThreads, 2); const compatibleBinding = { ...binding, @@ -546,11 +557,12 @@ describe('NativeFullTextDerivedIndexLifecycle', () => { binding.runtimeInfo = async () => ({ packageVersion: 'test', tantivyVersion: 'test', - nativeAbiVersion: 13, + nativeAbiVersion: 14, queryClassIsolationMinimumSearchThreads: 2, lifecycleApiVersion: 2, mutationBatchApiVersion: 5, - queryApiVersion: 7, + queryApiVersion: 8, + telemetryApiVersion: 1, storageBackends: ['native'], limits: { maxCommitPayloadBytes: 64 * 1024, @@ -578,7 +590,7 @@ describe('NativeFullTextDerivedIndexLifecycle', () => { binding.runtimeInfo = async () => ({ packageVersion: 'test', tantivyVersion: 'test', - nativeAbiVersion: 13, + nativeAbiVersion: 14, queryClassIsolationMinimumSearchThreads: 2, lifecycleApiVersion: 1, storageBackends: ['native'], @@ -601,6 +613,27 @@ describe('NativeFullTextDerivedIndexLifecycle', () => { await assert.rejects(lifecycle.initialize(), /incompatible runtime capabilities/); }); + for (const [capability, value] of [ + ['nativeAbiVersion', 13], + ['queryApiVersion', 7], + ['telemetryApiVersion', undefined], + ['telemetryApiVersion', 2], + ]) + it(`rejects incompatible ${capability} ${value}`, async () => { + const binding = new FakeNativeModule(); + const runtimeInfo = binding.runtimeInfo.bind(binding); + binding.runtimeInfo = async () => ({ ...(await runtimeInfo()), [capability]: value }); + await assert.rejects(validateFullTextNativeBinding(binding), /incompatible runtime capabilities/); + }); + + for (const className of ['NativeFullTextIndex', 'NativeFullTextReader']) + for (const method of ['telemetry', 'configureTelemetry']) + it(`rejects missing ${className}.${method}`, async () => { + const binding = new FakeNativeModule(); + delete binding[className].prototype[method]; + await assert.rejects(validateFullTextNativeBinding(binding), /required Harper binding contract/); + }); + it('rejects an unknown mutation-batch API version', async () => { const binding = new FakeNativeModule(); const runtimeInfo = binding.runtimeInfo.bind(binding); @@ -695,7 +728,7 @@ function upgradeNativeOptions(configuration, lifecycle) { }; } -describe('published full-text 0.5.0 and 0.7.0 to 0.8.0 migration', function () { +describe('published full-text 0.5.0, 0.7.0 and 0.8.0 to current migration', function () { this.timeout(60_000); let binding; let rootPath; @@ -707,42 +740,45 @@ describe('published full-text 0.5.0 and 0.7.0 to 0.8.0 migration', function () { assert.strictEqual((await binding.runtimeInfo()).packageVersion, fullTextPackageVersion); }); - it('reuses a genuine 0.7 filter-free checkpoint through 0.8 writes and 0.7 reopen', async () => { - const configuration = options(path.join(rootPath, 'native-v07-upgrade'), binding, { surfaceTerms: true }); - const lifecycle = new NativeFullTextDerivedIndexLifecycle(configuration); - await lifecycle.initialize({ reclaimRetired: false }); - const native = upgradeNativeOptions(configuration, lifecycle); - const payload = encodeFullTextCursorPayload({ format: 1, logs: { local: 42 } }); - previousFullTextProcess( - 'seed', - native, - [{ id: 'old', version: '1', fields: { title: 'catalog' } }], - payload, - '0.7.0' - ); - assert.deepStrictEqual(lifecycle.inspect(), { state: 'checkpointed', committedPayload: payload }); - const index = await lifecycle.open(); - try { - assert.deepStrictEqual( - (await index.search({ text: 'catalog' })).hits.map(({ id }) => id), - ['old'] + for (const previousVersion of ['0.7.0', '0.8.0']) + it(`reuses a genuine ${previousVersion} checkpoint through current writes and previous reopen`, async () => { + const configuration = options(path.join(rootPath, `native-${previousVersion}-upgrade`), binding, { + surfaceTerms: true, + }); + const lifecycle = new NativeFullTextDerivedIndexLifecycle(configuration); + await lifecycle.initialize({ reclaimRetired: false }); + const native = upgradeNativeOptions(configuration, lifecycle); + const payload = encodeFullTextCursorPayload({ format: 1, logs: { local: 42 } }); + previousFullTextProcess( + 'seed', + native, + [{ id: 'old', version: '1', fields: { title: 'catalog' } }], + payload, + previousVersion ); - await index.applyMutationBatch({ - upserts: [{ id: 'new', version: '2', fields: { title: 'catalog' } }], - deletes: [], + assert.deepStrictEqual(lifecycle.inspect(), { state: 'checkpointed', committedPayload: payload }); + const index = await lifecycle.open(); + try { + assert.deepStrictEqual( + (await index.search({ text: 'catalog' })).hits.map(({ id }) => id), + ['old'] + ); + await index.applyMutationBatch({ + upserts: [{ id: 'new', version: '2', fields: { title: 'catalog' } }], + deletes: [], + }); + await index.publish(encodeFullTextCursorPayload({ format: 1, logs: { local: 43 } })); + } finally { + await index.close({ mode: 'rollback' }); + } + const reopened = previousFullTextProcess('inspect', native, undefined, undefined, previousVersion); + assert.deepStrictEqual(reopened.inspection, { + state: 'checkpointed', + committedPayload: encodeFullTextCursorPayload({ format: 1, logs: { local: 43 } }), }); - await index.publish(encodeFullTextCursorPayload({ format: 1, logs: { local: 43 } })); - } finally { - await index.close({ mode: 'rollback' }); - } - const reopened = previousFullTextProcess('inspect', native, undefined, undefined, '0.7.0'); - assert.deepStrictEqual(reopened.inspection, { - state: 'checkpointed', - committedPayload: encodeFullTextCursorPayload({ format: 1, logs: { local: 43 } }), + assert.deepStrictEqual(reopened.searchIds, ['new', 'old']); + assert.strictEqual(reopened.openError, undefined); }); - assert.deepStrictEqual(reopened.searchIds, ['new', 'old']); - assert.strictEqual(reopened.openError, undefined); - }); it('rejects old checkpoints, retires them and refuses unsafe downgrade opens', async () => { const configuration = options(path.join(rootPath, 'native-upgrade-lifecycle'), binding, { surfaceTerms: true }); diff --git a/unitTests/resources/nativeFullTextTelemetry.test.js b/unitTests/resources/nativeFullTextTelemetry.test.js new file mode 100644 index 0000000000..1dbda3602e --- /dev/null +++ b/unitTests/resources/nativeFullTextTelemetry.test.js @@ -0,0 +1,407 @@ +require('../testUtils'); +const assert = require('node:assert'); +const { join } = require('node:path'); +const { setTimeout: delay } = require('node:timers/promises'); +const { setupTestDBPath } = require('../testUtils'); +const { waitFor } = require('../waitFor'); +const { supportsPublishedFullTextBinding } = require('./fullTextNativePlatform'); +const { loadFullTextNativeBinding } = require('#src/resources/indexes/fullTextNativeBinding'); +const { + FullTextTelemetryCollector, + resolveFullTextTelemetryPolicy, + fullTextNativeCounterDelta, + flattenFullTextTelemetry, +} = require('#src/resources/analytics/fullTextTelemetry'); + +const limits = { + indexingThreads: 2, + searchThreads: 2, + writerMemoryBytes: 64 * 1024 * 1024, + maxQueuedCommands: 16, + maxQueuedBytes: 64 * 1024 * 1024, + maxBatchBytes: 8 * 1024 * 1024, +}; + +describe('Published full-text telemetry collection', function () { + this.timeout(30_000); + let binding, + root, + sequence = 0, + collector, + handles; + before(async function () { + if (!(await supportsPublishedFullTextBinding())) this.skip(); + root = setupTestDBPath(); + binding = await loadFullTextNativeBinding(); + }); + beforeEach(() => { + collector = new FullTextTelemetryCollector(); + handles = []; + }); + afterEach(async () => { + collector.stop(); + for (const handle of handles.reverse()) await handle.close({ mode: 'rollback' }); + }); + async function writer() { + const options = { + path: join(root, `telemetry-${++sequence}`), + indexId: `telemetry-${sequence}`, + generation: 'g1', + fields: [{ name: 'title' }], + analyzer: 'english@2', + positions: true, + limits, + }; + const index = await binding.openNativeFullTextIndex(options); + handles.push(index); + await index.applyMutationBatch({ + upserts: [{ id: 'catalog', version: '1', fields: { title: 'catalog widget' } }], + deletes: [], + }); + await index.publish('checkpoint'); + return { index, options }; + } + async function reader(options) { + const handle = await binding.openNativeFullTextReader(options); + handles.push(handle); + return handle; + } + function row(role = 'reader') { + const rows = collector.takeRows(); + return rows.find((entry) => entry.method === role); + } + it('leaves collection unscheduled by default while real searches continue', async () => { + const { index } = await writer(); + collector.register(index, 'catalog.Product/search', 'writer'); + collector.collect(index); + assert.strictEqual(collector.collecting, false); + assert.strictEqual(collector.scheduled, false); + assert.deepStrictEqual(collector.takeRows(), []); + assert.strictEqual((await index.search({ text: 'widget' })).hits.length, 1); + assert.strictEqual(index.telemetry().native.histogramsEnabled, false); + }); + it('enables, pauses and resumes the same reader with cumulative gap counts and independent options', async () => { + const { options } = await writer(); + const handle = await reader(options); + collector.register(handle, 'catalog.Product/search', 'reader'); + collector.setOperatorPolicy({ enabled: true, histograms: true }); + await handle.search({ text: 'widget' }); + collector.collect(handle); + const first = row(); + assert.strictEqual(first.native_operations_search_completed, '1'); + assert.strictEqual(first.native_histogramsEnabled, true); + assert.strictEqual(first.native_snapshotBytesEnabled, false); + const histogramCount = handle.telemetry().native.operations.search.execution.histogram.count; + collector.setOperatorPolicy({ enabled: false }); + assert.strictEqual(collector.scheduled, false); + await handle.search({ text: 'widget' }); + assert.strictEqual(handle.telemetry().native.operations.search.execution.histogram.count, histogramCount); + collector.collect(handle); + assert.deepStrictEqual(collector.takeRows(), []); + collector.setOperatorPolicy({ enabled: true, snapshotBytes: true }); + await handle.search({ text: 'widget' }); + collector.collect(handle); + const resumed = row(); + assert.strictEqual(resumed.series, first.series); + assert.strictEqual(resumed.native_operations_search_completed, '3'); + assert.strictEqual(resumed.native_histogramsEnabled, false); + assert.strictEqual(resumed.native_snapshotBytesEnabled, true); + assert.strictEqual(resumed.native_reader_snapshotBytesPending, true); + assert.doesNotThrow(() => JSON.stringify(resumed)); + }); + it('unions removable consumer options without persisting consumer-only observations', async () => { + const { index } = await writer(); + collector.register(index, 'catalog.Product/search', 'writer'); + const samples = []; + const removeHistogram = collector.subscribe((sample) => samples.push(sample), { + histograms: true, + intervalMilliseconds: 1000, + }); + const removeBytes = collector.subscribe(() => {}, { snapshotBytes: true }); + assert.strictEqual(index.telemetry().native.histogramsEnabled, true); + assert.strictEqual(index.telemetry().native.snapshotBytesEnabled, true); + collector.collect(index); + assert.strictEqual(samples.length, 1); + assert.deepStrictEqual(collector.takeRows(), []); + removeHistogram(); + assert.strictEqual(index.telemetry().native.histogramsEnabled, false); + assert.strictEqual(index.telemetry().native.snapshotBytesEnabled, true); + removeBytes(); + assert.strictEqual(index.telemetry().native.snapshotBytesEnabled, false); + assert.strictEqual(collector.scheduled, false); + }); + it('stops scheduled callbacks on removal and contains a reentrant disable', async () => { + const { index } = await writer(); + collector.register(index, 'catalog.Product/search', 'writer'); + let calls = 0; + const remove = collector.subscribe(() => calls++, { intervalMilliseconds: 1000 }); + await waitFor(() => calls > 0, 5000); + remove(); + const stopped = calls; + await delay(1100); + assert.strictEqual(calls, stopped); + let removeSelf; + removeSelf = collector.subscribe(() => removeSelf()); + collector.collect(index); + assert.strictEqual(collector.collecting, false); + assert.strictEqual(collector.scheduled, false); + }); + it('preserves one logical series across concurrent reader churn and final close accounting', async () => { + const { options } = await writer(); + const first = await reader(options); + const second = await reader(options); + collector.setOperatorPolicy({ enabled: true }); + collector.register(first, 'catalog.Product/search', 'reader'); + collector.register(second, 'catalog.Product/search', 'reader'); + await first.search({ text: 'widget' }); + await second.search({ text: 'widget' }); + collector.collect(first); + collector.collect(second); + const before = row(); + assert.strictEqual(before.activeHandles, 2); + assert.strictEqual(before.native_operations_search_completed, '2'); + assert.strictEqual(before.native_reader_documents, '1'); + collector.beforeClose(first); + collector.collect(first); + await first.close(); + collector.afterClose(first); + const final = row(); + assert.strictEqual(final.native_operations_close_completed, '1'); + assert.strictEqual(final.series, before.series); + assert.strictEqual(final.native_operations_search_completed, '2'); + assert.strictEqual(final.activeHandles, 1); + collector.register(first, 'catalog.Product/search', 'reader'); + await second.search({ text: 'widget' }); + collector.collect(second); + assert.strictEqual(row().native_operations_search_completed, '3'); + const third = await reader(options); + collector.register(third, 'catalog.Product/search', 'reader'); + await third.search({ text: 'widget' }); + collector.collect(third); + const current = row(); + assert.strictEqual(current.series, before.series); + assert.strictEqual(current.native_operations_search_completed, '4'); + }); + it('contains rejected asynchronous consumers with one callback in flight', async () => { + const { index } = await writer(); + collector.register(index, 'catalog.Product/search', 'writer'); + let calls = 0; + let reject; + const remove = collector.subscribe(() => { + calls++; + return new Promise((_, fail) => { + reject = fail; + }); + }); + collector.collect(index); + collector.collect(index); + assert.strictEqual(calls, 1); + reject(new Error('consumer failed')); + await Promise.resolve(); + await Promise.resolve(); + collector.collect(index); + assert.strictEqual(calls, 2); + remove(); + reject(new Error('removed consumer failed')); + await Promise.resolve(); + }); + it('contains a real closed-handle configuration failure without poisoning other handles', async () => { + const { index } = await writer(); + await index.close(); + collector.register(index, 'catalog.Product/search', 'writer'); + const errors = []; + const remove = collector.subscribe((sample) => errors.push(sample.error)); + assert.deepStrictEqual(errors, ['configuration-error']); + const { index: live } = await writer(); + collector.register(live, 'catalog.Other/search', 'writer'); + assert.strictEqual((await live.search({ text: 'widget' })).hits.length, 1); + collector.collect(live); + assert.strictEqual(errors.at(-1), undefined); + remove(); + }); + it('bounds pending rows and marks dropped observations explicitly', async () => { + collector.stop(); + collector = new FullTextTelemetryCollector({ maxPendingRows: 2 }); + collector.setOperatorPolicy({ enabled: true }); + for (let i = 0; i < 3; i++) { + const { index } = await writer(); + collector.register(index, `catalog.Product${i}/search`, 'writer'); + collector.collect(index); + } + const rows = collector.takeRows(); + assert.strictEqual(rows.length, 2); + assert.strictEqual(rows.at(-1).droppedRows, '1'); + }); + it('uses exact uint64 deltas above safe integer range and across wrap', () => { + const previous = (1n << 63n) + 1n; + assert.strictEqual(fullTextNativeCounterDelta(previous + 3n, previous), 3n); + assert.strictEqual(fullTextNativeCounterDelta(2n, (1n << 64n) - 2n), 4n); + }); + it('preserves exact snapshot scalars and unavailable native observations without inventing zeros', async () => { + const { index } = await writer(); + const snapshot = index.telemetry(); + const huge = (1n << 63n) + 17n; + snapshot.native.operations.search.completed = huge; + snapshot.native.phases.commit.sumNanoseconds = huge; + const encoded = flattenFullTextTelemetry(snapshot); + assert.strictEqual(encoded.counters.get('native_operations_search_completed'), huge); + assert.strictEqual(encoded.counters.get('native_phases_commit_sumNanoseconds'), huge); + assert.strictEqual(encoded.gauges.native_reader_documents, '1'); + assert.strictEqual(encoded.gauges.native_reader_snapshotBytes, null); + const { native: _native, ...facade } = snapshot; + const unavailable = flattenFullTextTelemetry({ ...facade, available: false, reason: 'telemetry unavailable' }); + assert.strictEqual(unavailable.gauges.available, false); + assert.strictEqual(unavailable.gauges.native_reader_documents, undefined); + assert.strictEqual(unavailable.counters.get('logicalBatches_completed'), 1n); + }); + it('validates optional configuration without adding an enabled default', () => { + assert.strictEqual(resolveFullTextTelemetryPolicy(undefined).enabled, false); + for (const value of [ + null, + true, + [], + { enabled: 'true' }, + { intervalMilliseconds: 0 }, + { intervalMilliseconds: 2147483648 }, + { histograms: 1 }, + { unexpected: true }, + ]) + assert.throws(() => resolveFullTextTelemetryPolicy(value), TypeError); + }); +}); + +describe('Harper full-text analytics lifecycle and persistence', function () { + this.timeout(45_000); + it('collects existing writer/readers after live opt-in, persists exact rows, pauses and resumes without reopening', async function () { + if (process.env.HARPER_STORAGE_ENGINE === 'lmdb' || !(await supportsPublishedFullTextBinding())) this.skip(); + const fs = require('node:fs'); + const YAML = require('yaml'); + const analytics = require('#src/resources/analytics/write'); + const analyticsRead = require('#src/resources/analytics/read'); + const { table, databases } = require('#src/resources/databases'); + const { fullTextDerivedIndexReadiness } = require('#src/resources/derivedIndexes'); + const { setMainIsWorker } = require('#src/server/threads/manageThreads'); + const { getConfigFilePath } = require('#src/config/configUtils'); + const { fullTextTelemetryCollector: live } = require('#src/resources/analytics/fullTextTelemetryRuntime'); + setupTestDBPath(); + setMainIsWorker(true); + const file = getConfigFilePath(); + const original = fs.readFileSync(file, 'utf8'); + const config = YAML.parse(original); + const wasEnabled = analytics.checkAnalyticsEnabled(); + let Product; + const label = `telemetry-live-${Date.now()}`; + function change(enabled) { + fs.writeFileSync( + file, + YAML.stringify({ + ...config, + analytics: { ...config.analytics, fullText: { enabled, intervalMilliseconds: 1000 } }, + }) + ); + } + function rows() { + const found = []; + const raw = databases.system?.hdb_raw_analytics; + if (raw) + for (const { value } of raw.primaryStore.getRange({ start: false, end: Infinity })) + for (const row of value?.metrics ?? []) + if (row.metric === 'fulltext' && row.path.includes(label)) found.push(row); + return found; + } + async function search() { + const found = []; + for await (const record of Product.search({ + conditions: [ + { + attribute: 'catalogSearch', + comparator: 'matches', + value: 'widget', + maxIndexLagMilliseconds: 0, + waitForIndexMilliseconds: 5000, + }, + ], + })) + found.push(record.id); + return found.sort(); + } + try { + analytics.setAnalyticsEnabled(true); + Product = table({ + database: label, + table: 'Product', + audit: true, + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'title', type: 'String' }, + ], + fullTextIndexes: [ + { + name: 'catalogSearch', + fields: [{ name: 'title', weight: 1 }], + analyzer: 'english@2', + stopWords: true, + positions: true, + surfaceTerms: true, + synonyms: [], + }, + ], + }); + await Product.put('one', { title: 'catalog widget' }); + await waitFor(() => fullTextDerivedIndexReadiness(Product, 'catalogSearch').state === 'ready', 30000); + assert.deepStrictEqual(await search(), ['one']); + assert.strictEqual(live.collecting, false); + assert.strictEqual(live.scheduled, false); + assert.deepStrictEqual(rows(), []); + change(true); + await waitFor(() => live.collecting, 5000); + await waitFor( + () => rows().some((row) => row.method === 'writer') && rows().some((row) => row.method === 'reader'), + 10000 + ); + const writer = rows().find((row) => row.method === 'writer'); + assert(BigInt(writer.logicalBatches_completed) >= 1n); + assert.strictEqual(writer.native_histogramsEnabled, false); + assert.strictEqual(writer.native_snapshotBytesEnabled, false); + await waitFor(async () => { + await analytics.runAggregationCycle(1000, 1); + const found = []; + for await (const row of await analyticsRead.getOp({ + metric: 'fulltext', + conditions: [{ attribute: 'path', comparator: 'equals', value: writer.path }], + })) + found.push(row); + return found.some( + (row) => row.series === writer.series && row.logicalBatches_completed === writer.logicalBatches_completed + ); + }, 10000); + change(false); + await waitFor(() => !live.collecting && !live.scheduled, 5000); + const count = rows().length; + await Product.put('two', { title: 'second widget' }); + assert.deepStrictEqual(await search(), ['one', 'two']); + await delay(2200); + assert.strictEqual(rows().length, count); + change(true); + await waitFor(() => live.collecting, 5000); + await waitFor( + () => + rows().some( + (row) => + row.method === 'writer' && + row.series === writer.series && + BigInt(row.logicalBatches_completed) > BigInt(writer.logicalBatches_completed) + ), + 10000 + ); + } finally { + fs.writeFileSync(file, original); + await waitFor(() => !live.collecting, 5000); + if (Product) { + await Product.fullTextQueryIndexes.catalogSearch.customIndex.close(); + await Product.derivedIndexRuntime?.close(); + } + analytics.setAnalyticsEnabled(wasEnabled); + } + }); +}); diff --git a/utility/environment/environmentManager.ts b/utility/environment/environmentManager.ts index 8b6969a9c8..6258bef8bd 100644 --- a/utility/environment/environmentManager.ts +++ b/utility/environment/environmentManager.ts @@ -192,6 +192,49 @@ function sameConfigValue(a: any, b: any) { } } +/** Refresh one live-config subtree without losing unrelated installer or inherited overrides. */ +export function refreshConfigSubtree( + param: string, + previousRoot: unknown, + nextRoot: unknown, + validate: (value: unknown) => unknown +): unknown { + const canonical = (hdbTerms.CONFIG_PARAM_MAP as any)[param.toLowerCase()] ?? param; + const target = canonical.split('_'); + const read = (root: any, parts: string[]) => { + for (const part of parts) root = root && typeof root === 'object' ? root[part] : undefined; + return root; + }; + let value = snapshotConfigValue(read(nextRoot, target)); + const retired: string[] = []; + for (const [name, override] of appliedOverrides) { + const parts = name.split('_'); + if (!parts.slice(0, target.length).every((part, i) => part === target[i])) continue; + if (previousRoot !== undefined && !sameConfigValue(read(previousRoot, parts), read(nextRoot, parts))) { + retired.push(name); + continue; + } + if (parts.length <= target.length) value = snapshotConfigValue(read(override.value, target.slice(parts.length))); + else { + if (!value || typeof value !== 'object' || Array.isArray(value)) value = {}; + let node = value; + const relative = parts.slice(target.length); + for (const part of relative.slice(0, -1)) { + if (!node[part] || typeof node[part] !== 'object' || Array.isArray(node[part])) node[part] = {}; + node = node[part]; + } + node[relative.at(-1)!] = snapshotConfigValue(override.value); + } + } + const validated = validate(value); + for (const name of retired) appliedOverrides.delete(name); + configUtils.updateConfigObject(canonical, value); + for (const name of new Set(Object.values(hdbTerms.CONFIG_PARAMS))) + if (name.startsWith(canonical + '_')) + configUtils.updateConfigObject(name, read(value, name.slice(canonical.length + 1).split('_'))); + return validated; +} + /** * Replays this thread's overrides on top of a config that was just re-read from disk. A forced * initSync() rebuilds the whole config, which would otherwise revert the thread to the on-disk diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index 156da14482..2203bb73f2 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -522,6 +522,11 @@ export const LEGACY_CONFIG_PARAMS = { */ export const CONFIG_PARAMS = { ANALYTICS_AGGREGATEPERIOD: 'analytics_aggregatePeriod', + ANALYTICS_FULL_TEXT: 'analytics_fullText', + ANALYTICS_FULL_TEXT_ENABLED: 'analytics_fullText_enabled', + ANALYTICS_FULL_TEXT_INTERVAL_MILLISECONDS: 'analytics_fullText_intervalMilliseconds', + ANALYTICS_FULL_TEXT_HISTOGRAMS: 'analytics_fullText_histograms', + ANALYTICS_FULL_TEXT_SNAPSHOT_BYTES: 'analytics_fullText_snapshotBytes', ANALYTICS_AGGREGATERETENTIONMS: 'analytics_aggregateRetentionMs', ANALYTICS_RAWRETENTIONMS: 'analytics_rawRetentionMs', ANALYTICS_REPLICATE: 'analytics_replicate', diff --git a/validation/configValidator.ts b/validation/configValidator.ts index a75c043c8c..2e668860b1 100644 --- a/validation/configValidator.ts +++ b/validation/configValidator.ts @@ -443,6 +443,12 @@ export function configValidator(configJson, skipFsValidation = false) { analytics: Joi.object({ aggregatePeriod: number, replicate: boolean.optional(), + fullText: Joi.object({ + enabled: boolean.optional(), + intervalMilliseconds: number.integer().min(1000).max(2147483647).optional(), + histograms: boolean.optional(), + snapshotBytes: boolean.optional(), + }).optional(), }), replication: Joi.object({ hostname: bareHostConstraints, From 64310450547ef1f1788f9c7c892098c706bace21 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Mon, 2 Nov 2026 22:07:47 -0700 Subject: [PATCH 02/26] Preserve full-text telemetry through lifecycle and policy changes --- .github/workflows/unit-test.yml | 2 + AGENTS.md | 2 +- DESIGN.md | 2 +- benchmarks/fulltext-analytics.bench.js | 3 +- benchmarks/fulltext-analytics.mjs | 49 ++++-- config/DESIGN.md | 2 +- config/fullTextTelemetryConfiguration.ts | 9 +- resources/FULL_TEXT.md | 8 +- resources/analytics/DESIGN.md | 2 +- resources/analytics/fullTextTelemetry.ts | 68 ++++++-- .../analytics/fullTextTelemetryRuntime.ts | 16 +- resources/analytics/write.ts | 18 ++- resources/indexes/fullTextDerivedIndex.ts | 13 +- .../fullTextTelemetryConfiguration.test.js | 22 ++- .../fixtures/fullTextTelemetryWorker.cjs | 92 +++++++++++ .../resources/nativeFullTextTelemetry.test.js | 107 ++++++++++-- .../nativeFullTextTelemetryClose.test.js | 116 +++++++++++++ .../nativeFullTextTelemetryWorkers.test.js | 152 ++++++++++++++++++ 18 files changed, 611 insertions(+), 72 deletions(-) create mode 100644 unitTests/resources/fixtures/fullTextTelemetryWorker.cjs create mode 100644 unitTests/resources/nativeFullTextTelemetryClose.test.js create mode 100644 unitTests/resources/nativeFullTextTelemetryWorkers.test.js diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 2f5a479beb..d0013edba6 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -332,6 +332,8 @@ jobs: run: >- npx mocha unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js unitTests/resources/nativeFullTextTelemetry.test.js + unitTests/resources/nativeFullTextTelemetryClose.test.js + unitTests/resources/nativeFullTextTelemetryWorkers.test.js unitTests/resources/nativeFullTextTableSearch.test.js unitTests/resources/nativeFullTextIndexSchemaUpgrade.test.js unitTests/resources/nativeFullTextIndexUpgradeWorkers.test.js diff --git a/AGENTS.md b/AGENTS.md index f24bba7f73..4d792de2bc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -122,7 +122,7 @@ Use this to land in the right folder before grepping. Every top-level folder is ### Non-source - **`bin/`** — covered above (it's source). -- **`benchmarks/`** — HNSW vector-search benchmark only (`hnsw-search.js`). Stand-alone; not part of CI. +- **`benchmarks/`** — Stand-alone HNSW search (`hnsw-search.js`) and paired full-text analytics (`fulltext-analytics.mjs` / `.bench.js`) workloads; not part of CI. - **`build-tools/`** — build-pipeline scripts. [build-tools/DESIGN.md](build-tools/DESIGN.md) covers the published artifacts. Tests: `unitTests/build-tools/`; run `npm run test:unit:main` after changes. - **`dev/`** — single dev utility (`sync-commits.js`) for cross-repo commit syncing. Not runtime. - **`integrationTests/`** — end-to-end tests against a built distribution. Run with `npm run test:integration` / `npm run test:integration:all`. Subdirs mirror source. See `integrationTests/README.md`. diff --git a/DESIGN.md b/DESIGN.md index d75b6edcb6..fccae14a71 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -78,7 +78,7 @@ A note lives in the `DESIGN.md` of the directory that owns the code it describes ## resources/analytics/ -- [Full-text collection follows actual handle ownership](resources/analytics/DESIGN.md#full-text-collection-follows-actual-handle-ownership) — Opt-in. +- [Full-text handle ownership](resources/analytics/DESIGN.md#full-text-collection-follows-actual-handle-ownership) — Opt-in; counters survive reader churn. - [Analytics aggregation resumes from a raw cursor, which is not a clock (`resources/analytics/write.ts`)](resources/analytics/DESIGN.md#analytics-aggregation-resumes-from-a-raw-cursor-which-is-not-a-clock-resourcesanalyticswritets) — `rawCursor` moves only to a record a cycle actually read and is separate from the cadence marker. - [Per-thread rows carry a `thread` attribute under their own metric name (`resources/analytics/write.ts`)](resources/analytics/DESIGN.md#per-thread-rows-carry-a-thread-attribute-under-their-own-metric-name-resourcesanalyticswritets) — `perThread` entries roll up one row per thread; `duration` per thread is `duration-by-thread`, never extra `duration` rows; rows stay flat scalars for the Grafana datasource. · [A reset ends the window that was being flushed too (`resources/analytics/write.ts`)](resources/analytics/DESIGN.md#a-reset-ends-the-window-that-was-being-flushed-too-resourcesanalyticswritets) — `setAnalyticsEnabled()` advances a generation; a flush that sees it change stops delivering. - [Aggregation folds each measure over only the samples that carry it (`resources/analytics/write.ts`)](resources/analytics/DESIGN.md#aggregation-folds-each-measure-over-only-the-samples-that-carry-it-resourcesanalyticswritets) — per-measure sample counts, cycle-local, weight each mean; a measure in every sample folds exactly as before. diff --git a/benchmarks/fulltext-analytics.bench.js b/benchmarks/fulltext-analytics.bench.js index 22a019982a..719981712b 100644 --- a/benchmarks/fulltext-analytics.bench.js +++ b/benchmarks/fulltext-analytics.bench.js @@ -28,7 +28,7 @@ it(`qualifies actual Harper fulltext analytics (${configuration.profile})`, asyn await ensureSystemTables(); const binding = await loadFullTextNativeBinding(); const runtime = await binding.runtimeInfo(); - assert.strictEqual(runtime.packageVersion, '0.9.0'); + assert.strictEqual(runtime.packageVersion, require('../package.json').optionalDependencies['@harperfast/fulltext']); const path = getConfigFilePath(); const savedConfig = readFileSync(path, 'utf8'); const products = []; @@ -229,6 +229,7 @@ it(`qualifies actual Harper fulltext analytics (${configuration.profile})`, asyn cpu, eventLoop: { utilization: utilization.utilization, + samples: eventLoop.count, meanDelayMilliseconds: eventLoop.mean / 1e6, maxDelayMilliseconds: eventLoop.max / 1e6, p99DelayMilliseconds: eventLoop.percentile(99) / 1e6, diff --git a/benchmarks/fulltext-analytics.mjs b/benchmarks/fulltext-analytics.mjs index 52c8e3cf08..295fa33087 100644 --- a/benchmarks/fulltext-analytics.mjs +++ b/benchmarks/fulltext-analytics.mjs @@ -1,5 +1,4 @@ -import { spawn } from 'node:child_process'; -import { execFileSync } from 'node:child_process'; +import { spawn, execFileSync } from 'node:child_process'; import { mkdirSync, writeFileSync, readFileSync, createWriteStream, existsSync, readdirSync } from 'node:fs'; import { resolve, dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -24,7 +23,7 @@ for (let i = 2; i < process.argv.length; i += 2) { options[name] = number; } } -options.output ??= `/Users/kyle/WebstormProjects/tmp/fulltext-analytics-${new Date().toISOString().replace(/[:.]/g, '-')}`; +if (!options.output) throw new Error('An explicit --output directory is required'); if (existsSync(options.output) && readdirSync(options.output).length) throw new Error('Use a fresh or empty --output directory to preserve prior qualification evidence'); mkdirSync(options.output, { recursive: true }); @@ -42,6 +41,10 @@ const fingerprintFiles = [ 'resources/analytics/fullTextTelemetry.ts', 'resources/analytics/fullTextTelemetryRuntime.ts', 'config/fullTextTelemetryConfiguration.ts', + 'dist/resources/analytics/fullTextTelemetry.js', + 'dist/resources/analytics/fullTextTelemetryRuntime.js', + 'dist/resources/analytics/write.js', + 'dist/resources/indexes/fullTextNativeBinding.js', ]; source.fileSha256 = Object.fromEntries( fingerprintFiles.map((file) => [ @@ -84,22 +87,46 @@ async function spawnMocha(profile, pair) { ], { cwd: repository, env: childEnv, stdio: ['ignore', 'pipe', 'pipe'] } ); - child.stdout.pipe(log, { end: false }); - child.stderr.pipe(log, { end: false }); + const MAX_LOG_BYTES = 2 * 1024 * 1024; + let logBytes = 0; + let droppedLogBytes = 0; + let logError; + log.once('error', (error) => { + logError = error.code ?? error.message; + child.kill('SIGTERM'); + }); + function capture(chunk) { + const retained = chunk.subarray(0, Math.max(0, MAX_LOG_BYTES - logBytes)); + if (retained.length && !log.destroyed) log.write(retained); + logBytes += retained.length; + droppedLogBytes += chunk.length - retained.length; + } + child.stdout.on('data', capture); + child.stderr.on('data', capture); let timedOut = false; + let killTimer; const timer = setTimeout(() => { timedOut = true; child.kill('SIGTERM'); + killTimer = setTimeout(() => child.kill('SIGKILL'), 5000).unref(); }, options.duration + 120_000); - const result = await new Promise((resolveChild, reject) => { - child.once('error', reject); - child.once('exit', (code, signal) => resolveChild({ code, signal })); + const result = await new Promise((resolveChild) => { + child.once('error', (error) => resolveChild({ code: null, signal: null, spawnError: error.message })); + child.once('close', (code, signal) => resolveChild({ code, signal })); }).finally(() => { clearTimeout(timer); - log.end(); + clearTimeout(killTimer); }); - writeFileSync(join(directory, 'launcher-result.json'), JSON.stringify({ ...result, timedOut }, null, 2)); - if (result.code !== 0 || timedOut || !existsSync(join(directory, 'result.json'))) + await new Promise((resolveLog) => (log.destroyed ? resolveLog() : log.end(resolveLog))); + writeFileSync( + join(directory, 'launcher-result.json'), + JSON.stringify( + { ...result, timedOut, logError, logBytes, droppedLogBytes, maximumLogBytes: MAX_LOG_BYTES }, + null, + 2 + ) + ); + if (result.code !== 0 || timedOut || logError || !existsSync(join(directory, 'result.json'))) throw new Error(`${profile} pair ${pair} failed; retained ${directory}`); return JSON.parse(readFileSync(join(directory, 'result.json'), 'utf8')); } diff --git a/config/DESIGN.md b/config/DESIGN.md index f7128e1328..bf2480a239 100644 --- a/config/DESIGN.md +++ b/config/DESIGN.md @@ -240,7 +240,7 @@ off on the very boot that could not read its configuration. ## `RootConfigWatcher` schedules a trailing verify-read (`config/RootConfigWatcher.ts`) -Chokidar throttles `EV.CHANGE` events per path for 50ms (`node_modules/chokidar/index.js`) with no trailing emission, so the second of two `atomicWriteFile()` renames landing within 50ms produces no watcher event. After a successful `handleChange` read, `RootConfigWatcher` arms one 100ms (`TRAILING_VERIFY_READ_DELAY_MS`, double the 50ms throttle window) trailing verify-read (`#read(true)`), re-armed on each event and cleared on `close()`, terminal read failure, or when `ConfigReadRetry` takes over. Post-`ready` reads short-circuit when raw file text matches `#lastData` and otherwise compare the parsed object against an unmutated clone (`#lastParsedConfig`) with `isDeepStrictEqual`, so neither the trailing read nor an identical rewrite emits a duplicate `change` even when consumers (`harper_logger`) mutate `this.#config` in place (`unitTests/config/rootConfigWatcher.test.js`). Root-config `OptionsWatcher`s do not schedule a trailing read and remain exposed to the 50ms throttle window. +Chokidar throttles `EV.CHANGE` events per path for 50ms (`node_modules/chokidar/index.js`) with no trailing emission, so the second of two `atomicWriteFile()` renames landing within 50ms produces no watcher event. After a successful `handleChange` read, `RootConfigWatcher` arms one 100ms (`TRAILING_VERIFY_READ_DELAY_MS`, double the 50ms throttle window) trailing verify-read (`#read(true)`), re-armed on each event and cleared on `close()`, terminal read failure, or when `ConfigReadRetry` takes over. Post-`ready` reads short-circuit when raw file text matches `#lastData` and otherwise compare the parsed object against an unmutated clone (`#lastParsedConfig`) with `isDeepStrictEqual`, so neither the trailing read nor an identical rewrite emits a duplicate `change` even when consumers (`harper_logger`) mutate `this.#config` in place (`unitTests/config/rootConfigWatcher.test.js`). Root-config `OptionsWatcher`s lack this trailing read. `unlink` emits contained `unavailable`; the first successful recovery read emits `available` even when file text is unchanged. Telemetry uses these events to pause file-based policy and reapply environment overlays; invalid recreation retains the last valid policy (`unitTests/config/fullTextTelemetryConfiguration.test.js`). ## `RootConfigWatcher` polls under Bun (`config/RootConfigWatcher.ts`) diff --git a/config/fullTextTelemetryConfiguration.ts b/config/fullTextTelemetryConfiguration.ts index 0df2fa3813..6aae2e1c86 100644 --- a/config/fullTextTelemetryConfiguration.ts +++ b/config/fullTextTelemetryConfiguration.ts @@ -4,11 +4,11 @@ import { overlayRootEnvConfig } from './harperConfigEnvVars.ts'; import { get, refreshConfigSubtree } from '../utility/environment/environmentManager.ts'; import { resolveFullTextTelemetryPolicy } from '../resources/analytics/fullTextTelemetry.ts'; import { loggerWithTag } from '../utility/logging/logger.ts'; +import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; const logger = loggerWithTag('fulltext-telemetry'); -const PARAM = 'analytics_fullText'; +const PARAM = CONFIG_PARAMS.ANALYTICS_FULL_TEXT; -/** Observe only telemetry settings; invalid edits retain the last valid policy. */ export function watchFullTextTelemetryConfiguration( apply: (value: unknown) => void, watcher: RootConfigWatcher = getSharedRootConfigWatcher() @@ -41,8 +41,9 @@ export function watchFullTextTelemetryConfiguration( function unavailable() { if (stopped) return; try { - applyPolicy(refreshConfigSubtree(PARAM, previousRoot, {}, resolveFullTextTelemetryPolicy)); - previousRoot = {}; + const root = structuredClone(overlayRootEnvConfig({}) ?? {}); + applyPolicy(refreshConfigSubtree(PARAM, previousRoot, root, resolveFullTextTelemetryPolicy)); + previousRoot = root; } catch { report(); } diff --git a/resources/FULL_TEXT.md b/resources/FULL_TEXT.md index 0b7539932a..3115e64edf 100644 --- a/resources/FULL_TEXT.md +++ b/resources/FULL_TEXT.md @@ -207,14 +207,14 @@ analytics: The implied interval is 60,000 milliseconds per handle. Calls are staggered across existing writer/readers, and their rows use Harper's existing analytics flush, storage, retention and protected `get_analytics` interface. Collection never opens a cold index. To tune it, set `intervalMilliseconds` between 1,000 and 2,147,483,647; synchronous snapshots and export work have measurable cost, so qualify the intended number of indexes and cadence. No production overhead bound is claimed. -Set `enabled: false` or remove the block to stop scheduled polling and persistent full-text export. A synchronous observation already executing completes. Editing unrelated settings does not restart collection. Invalid telemetry edits retain the last valid policy; deletion of the configuration file pauses operator collection until a valid file returns. Environment overlays retain their existing precedence. Disabling analytics also disables its full-text collection demand. Internal registered consumers can independently request observations; removing the last demand stops the timer. Consumer-only observations do not enable persistent analytics. +Set `enabled: false` or remove the block to stop scheduled polling and persistent full-text export. A synchronous observation already executing completes. Editing unrelated settings does not restart collection. Invalid telemetry edits retain the last valid policy; deletion of the configuration file pauses file-based operator collection; environment overlays still apply while the file is missing. Disabling analytics also disables its full-text collection demand. Internal registered consumers can independently request observations; removing the last demand stops the timer. Consumer-only observations do not enable persistent analytics. -`histograms: true` and `snapshotBytes: true` independently enable optional recording/reader-byte observation; both default to false. Turning collection off explicitly disables options no remaining consumer needs. Re-enabling resumes cumulative counters. Enabling bytes schedules no native publication: values can remain pending until a successful publication/reload opportunity. Cached bytes, observation age and errors retain their own meanings; reader bytes are not directory totals or resident memory. +`histograms: true` and `snapshotBytes: true` independently enable optional recording/reader-byte observation; both default to false. Turning collection off disables options no remaining consumer needs on live handles; a handle already closing keeps its last options until close settles. Re-enabling resumes cumulative counters. Enabling bytes schedules no native publication: values can remain pending until a successful publication/reload opportunity. Cached bytes, observation age and errors retain their own meanings; reader bytes are not directory totals or resident memory. -Read `get_analytics` with `metric: fulltext`. Each stored row identifies `path` (database/table/index), `method` (`writer` or `reader`), worker `thread`, a bounded-lifetime `series` reset marker and actual `sampledAt`. Field names flatten the wrapper snapshot: for example, `native_operations_search_completed`, `native_operations_search_queue_sumNanoseconds`, `native_queues_ordinarySearch_commands` and `logicalBatches_completed`. Values are flat JSON scalars; exact bigint counters, nanoseconds, bytes, histogram counts and bounds are decimal strings. Null and availability/pending/error flags mean unavailable, not zero. Histograms retain noncumulative buckets; no percentiles are invented. +Read `get_analytics` with `metric: fulltext`. Each stored row identifies `path` (`./`, or `index:` when the label exceeds 256 bytes), `method` (`writer` or `reader`), worker `thread`, a bounded-lifetime `series` reset marker and actual `sampledAt`. Field names flatten the wrapper snapshot: for example, `native_operations_search_completed`, `native_operations_search_queue_sumNanoseconds`, `native_queues_ordinarySearch_commands` and `logicalBatches_completed`. Values are flat JSON scalars; exact bigint counters, nanoseconds, bytes, histogram counts and bounds are decimal strings. Availability/pending/error flags distinguish missing observations from zero. Null can mean an unavailable observation, an empty queue's oldest age, or a histogram's unbounded final bucket. Histograms retain noncumulative buckets; no percentiles are invented. Histogram deltas cover only intervals with recording enabled, not an entire collection gap when recording was disabled. Native uint64 counter wraps are unwrapped into exact logical-series totals. Reader reopen and overlapping retired readers carry counters into the same worker/index/role series; document/segment/generation observations come from the most recently observed eligible reader rather than summing duplicate snapshots. Queue occupancy sums live handles of the same role; oldest age uses their maximum. Shared queue capacities overlap: never add ordinary/expensive capacities or treat them as a process-wide budget. Final close counters are sampled once after settlement; no recurring scrape runs while close is in flight. -Rollups keep the newest cumulative row per worker/index/role instead of averaging counters. For rates, decode decimal counters with `BigInt`, require the same `series` and complete/available observations, subtract exact totals and divide by actual `sampledAt` elapsed time. Check integer safety before converting a delta to `number`. A new series, unavailable samples or incomplete coverage require a new baseline; never report a missing interval as zero activity. `complete: false` flags lost terminal coverage; `observationAgeMilliseconds` distinguishes retained observations from the latest attempt. Pending rows are coalesced per logical series in a 128-row buffer; overflow drops the oldest pending row and increments exact `droppedRows`. Historical rows follow configured analytics retention, not handle lifetime. +Rollups keep the newest cumulative row per worker/index/role instead of averaging counters. For rates, decode decimal counters with `BigInt`, require the same `series` and complete/available observations, subtract exact totals and divide by actual `sampledAt` elapsed time. Check integer safety before converting a delta to `number`. A new series, unavailable samples or incomplete coverage require a new baseline; never report a missing interval as zero activity. `complete: false` flags lost final handle counts. The next successful live observation starts a new series so later rates can use a new baseline; retained absolute totals need not reset to zero. Dropping a pending observation does not invalidate cumulative totals from active handles; `observationAgeMilliseconds` distinguishes retained observations from the latest attempt. Pending rows are coalesced per logical series in a 128-row buffer; overflow drops the oldest pending row and increments exact `droppedRows`. Historical rows follow configured analytics retention, not handle lifetime. Do not add facade logical batches to their native frame counts, sum overlapping legacy/native timing spans, or sum reader document counts across workers. Generation IDs are not source-log positions or freshness timestamps. Harper still owns readiness, source catch-up, authorization, hydration and end-to-end request attribution; this telemetry does not replace those checks. Queries, terms, record IDs, users, raw cursors, filesystem paths and raw exception messages are not exported labels. Detailed automatic merge statistics, cancellation and per-index RSS remain explicitly unsupported. diff --git a/resources/analytics/DESIGN.md b/resources/analytics/DESIGN.md index 62ff009505..4d688c3ab9 100644 --- a/resources/analytics/DESIGN.md +++ b/resources/analytics/DESIGN.md @@ -75,6 +75,6 @@ always a caller's measure. `fullTextTelemetry.ts` keeps private per-handle baselines and carries exact counters into a stable worker/index/role series. `nativeFullTextTelemetry.test.js` pins reader churn, wrap/precision, opt-in toggles and final close. The scheduler pauses before native close and observes final scalars only after settlement, so it never polls close in flight. Completed registrations release handle references; inactive groups remain only until their coalesced final row is drained. Pending export has a 128-row memory bound with explicit drop counts. -`fullTextTelemetryRuntime.ts` joins removable internal consumer demand with live operator policy. Only operator demand enables persistent analytics. No demand means no telemetry timer; no cold handles are opened. Counters are decimal strings, rows are flat and `cumulative` makes the aggregation fold retain the newest sample, including availability/error fields, per worker. Existing request metrics retain their averaging semantics. `aggregationCycle.test.js` verifies exact counters above 2^53 survive storage and `get_analytics`. +`fullTextTelemetryRuntime.ts` joins removable internal consumer demand with live operator policy. Only operator demand enables persistent analytics. No demand means no telemetry timer; no cold handles are opened. Counters are decimal strings, rows are flat and `cumulative` makes the aggregation fold retain the newest sample, including availability/error fields, per worker. Lost final handle counts force a new series on the next successful live observation; pending-row drops preserve cumulative totals. Failed writer closes resume collection because Harper may keep the engine installed. Existing request metrics retain their averaging semantics. `aggregationCycle.test.js` verifies exact counters above 2^53 survive storage and `get_analytics`. `watchFullTextTelemetryConfiguration` uses the shared root watcher, environment overlay and transactional `refreshConfigSubtree`; unrelated edits preserve sampling cadence, invalid edits preserve valid policy, and availability events pause/recover after file deletion. `fullTextTelemetryConfiguration.test.js` names the effective override and recovery controls. Native runtime/query policy remains frozen separately. diff --git a/resources/analytics/fullTextTelemetry.ts b/resources/analytics/fullTextTelemetry.ts index 1d0dc81e55..cd5afe6349 100644 --- a/resources/analytics/fullTextTelemetry.ts +++ b/resources/analytics/fullTextTelemetry.ts @@ -1,5 +1,6 @@ import { randomUUID } from 'node:crypto'; import type { ConfigureTelemetryOptions, FullTextTelemetry } from '@harperfast/fulltext/native'; +import { METRIC } from './metadata.ts'; export type FullTextTelemetryPolicy = Required & { enabled: boolean; @@ -71,7 +72,7 @@ type Entry = { warned: boolean; }; type Consumer = { - callback: (sample: FullTextTelemetrySample) => void | Promise; + callback: (sample: FullTextTelemetrySample) => void | PromiseLike; policy: FullTextTelemetryPolicy; busy: boolean; }; @@ -155,7 +156,7 @@ export class FullTextTelemetryCollector { } subscribe( - callback: (sample: FullTextTelemetrySample) => void | Promise, + callback: (sample: FullTextTelemetrySample) => void | PromiseLike, options: Omit, 'enabled'> = {} ): () => void { if (typeof callback !== 'function') throw new TypeError('A full-text telemetry consumer must be a function'); @@ -227,12 +228,32 @@ export class FullTextTelemetryCollector { beforeClose(handle: object): void { const entry = this.#registered.get(handle); - if (entry) this.#registration(entry).beforeClose(); + if (entry) this.#closeHook(() => this.#registration(entry).beforeClose()); } afterClose(handle: object): void { const entry = this.#registered.get(handle); - if (entry) this.#registration(entry).afterClose(); + if (entry) this.#closeHook(() => this.#registration(entry).afterClose()); + } + + closeFailed(handle: object): void { + const entry = this.#registered.get(handle); + if (!entry) return; + entry.closing = false; + this.#closeHook(() => { + if (this.collecting) this.#observe(entry); + this.#schedule(); + }); + } + + #closeHook(action: () => void) { + try { + action(); + } catch { + try { + this.#onError(); + } catch {} + } } takeRows(): FullTextTelemetryRow[] { @@ -256,22 +277,36 @@ export class FullTextTelemetryCollector { } #updatePolicy() { - clearTimeout(this.#timer); - this.#timer = undefined; this.#revision++; const demands = [...this.#consumers].map((consumer) => consumer.policy); if (this.#operator.enabled) demands.push(this.#operator); - this.#policy = { + const previous = this.#policy; + const next = { enabled: demands.length > 0, - intervalMilliseconds: Math.min(...demands.map((policy) => policy.intervalMilliseconds), 2_147_483_647), + intervalMilliseconds: demands.length + ? Math.min(...demands.map((policy) => policy.intervalMilliseconds)) + : DEFAULT_FULL_TEXT_TELEMETRY_POLICY.intervalMilliseconds, histograms: demands.some((policy) => policy.histograms), snapshotBytes: demands.some((policy) => policy.snapshotBytes), }; + if ( + previous.enabled === next.enabled && + previous.intervalMilliseconds === next.intervalMilliseconds && + previous.histograms === next.histograms && + previous.snapshotBytes === next.snapshotBytes + ) { + this.#prune(); + return; + } + clearTimeout(this.#timer); + this.#timer = undefined; + this.#policy = next; + const recordingChanged = previous.histograms !== next.histograms || previous.snapshotBytes !== next.snapshotBytes; let offset = 0; const step = this.#policy.intervalMilliseconds / Math.max(1, this.#entries.size); for (const entry of this.#entries) { - entry.configured = false; - if (!entry.closing) this.#configure(entry); + if (recordingChanged) entry.configured = false; + if (!entry.closing && !entry.configured) this.#configure(entry); entry.nextAt = performance.now() + step * ++offset; } this.#prune(); @@ -320,6 +355,10 @@ export class FullTextTelemetryCollector { entry.observedAt = performance.now(); entry.warned = false; if (!snapshot.available && entry.closing) entry.group.complete = false; + if (snapshot.available && !entry.closing && !entry.group.complete) { + entry.group.series = randomUUID(); + entry.group.complete = true; + } this.#publish(entry, snapshot); } catch { this.#failed(entry, 'collection-error'); @@ -329,14 +368,14 @@ export class FullTextTelemetryCollector { #publish(entry: Entry, snapshot?: FullTextTelemetry, error?: FullTextTelemetrySample['error'], stage?: 'closing') { const sampledAt = Date.now(); const revision = this.#revision; - if (this.#operator.enabled) { + if (this.#operator.enabled && stage !== 'closing') { let newest = entry; for (const candidate of entry.group.entries) if (!candidate.closing && candidate.flat && (newest.closing || candidate.observedAt > newest.observedAt)) newest = candidate; const row: FullTextTelemetryRow = { ...newest.flat?.gauges, - metric: 'fulltext', + metric: METRIC.FULL_TEXT, path: entry.group.index, method: entry.group.role, perThread: true, @@ -346,7 +385,7 @@ export class FullTextTelemetryCollector { complete: entry.group.complete, collectionError: error ?? null, droppedRows: this.#droppedRows.toString(), - observationAgeMilliseconds: entry.observedAt ? performance.now() - entry.observedAt : null, + observationAgeMilliseconds: newest.observedAt ? performance.now() - newest.observedAt : null, activeHandles: [...entry.group.entries].filter((candidate) => !candidate.closing).length, closingHandles: [...entry.group.entries].filter((candidate) => candidate.closing).length, available: !error && snapshot?.available === true, @@ -373,7 +412,6 @@ export class FullTextTelemetryCollector { if (!this.#pending.has(entry.group) && this.#pending.size >= this.#maxPendingRows) { const oldest = this.#pending.keys().next().value!; this.#pending.delete(oldest); - oldest.complete = false; this.#droppedRows++; row.droppedRows = this.#droppedRows.toString(); this.#prune(); @@ -398,7 +436,7 @@ export class FullTextTelemetryCollector { }); if (result && typeof result.then === 'function') { consumer.busy = true; - void result + void Promise.resolve(result) .catch(() => { try { this.#onError(); diff --git a/resources/analytics/fullTextTelemetryRuntime.ts b/resources/analytics/fullTextTelemetryRuntime.ts index 3caee64ea7..67d9085f5c 100644 --- a/resources/analytics/fullTextTelemetryRuntime.ts +++ b/resources/analytics/fullTextTelemetryRuntime.ts @@ -17,13 +17,12 @@ let operatorPolicy = resolveFullTextTelemetryPolicy(undefined); let started = false; let stopWatching: (() => void) | undefined; let removeListener: (() => void) | undefined; -let enabledSince = 0; let persistenceActive = false; let nextWarningAt = 0; export const fullTextTelemetryCollector = new FullTextTelemetryCollector({ onPending: () => { - if (operatorPolicy.enabled && checkAnalyticsEnabled()) requestAnalyticsFlush(); + if (operatorPolicy.enabled && checkAnalyticsEnabled(false)) requestAnalyticsFlush(); }, onError: () => { if (Date.now() < nextWarningAt) return; @@ -34,14 +33,16 @@ export const fullTextTelemetryCollector = new FullTextTelemetryCollector({ export function setFullTextAnalyticsEnabled(enabled: boolean): void { const next = operatorPolicy.enabled && enabled; - if (next && !persistenceActive) enabledSince = Date.now(); persistenceActive = next; fullTextTelemetryCollector.setOperatorPolicy({ ...operatorPolicy, enabled: next }); } export function acceptsFullTextTelemetryRow(row: Record): boolean { return ( - persistenceActive && checkAnalyticsEnabled() && typeof row.sampledAt === 'number' && row.sampledAt >= enabledSince + persistenceActive && + checkAnalyticsEnabled(false) && + typeof row.sampledAt === 'number' && + Number.isFinite(row.sampledAt) ); } @@ -49,18 +50,19 @@ export function initializeFullTextTelemetry(): void { if (started) return; started = true; removeListener = addAnalyticsListener((metrics) => { + if (!persistenceActive || !checkAnalyticsEnabled(false)) return; const rows = fullTextTelemetryCollector.takeRows(); - if (operatorPolicy.enabled && checkAnalyticsEnabled()) metrics.push(...rows); + metrics.push(...rows); }); stopWatching = watchFullTextTelemetryConfiguration((value) => { const next = resolveFullTextTelemetryPolicy(value); operatorPolicy = next; - setFullTextAnalyticsEnabled(checkAnalyticsEnabled()); + setFullTextAnalyticsEnabled(checkAnalyticsEnabled(false)); }); } export function subscribeFullTextTelemetry( - callback: (sample: FullTextTelemetrySample) => void | Promise, + callback: (sample: FullTextTelemetrySample) => void | PromiseLike, options?: Omit, 'enabled'> ): () => void { initializeFullTextTelemetry(); diff --git a/resources/analytics/write.ts b/resources/analytics/write.ts index bb224c61a8..69dd477a6a 100644 --- a/resources/analytics/write.ts +++ b/resources/analytics/write.ts @@ -53,12 +53,13 @@ let analyticsGeneration = 0; onStartup(() => { analyticsEnabled = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) > -1; analyticsReadOnlyChecked = false; - setFullTextAnalyticsEnabled(checkAnalyticsEnabled()); + setFullTextAnalyticsEnabled(checkAnalyticsEnabled(false)); }); // Check read-only mode lazily to avoid circular dependency at module load time. The event loop // sampler starts here rather than at load, so a read-only worker never runs it. -export function checkAnalyticsEnabled(): boolean { +export function checkAnalyticsEnabled(startSampler = true): boolean { + if (!startSampler) return analyticsEnabled && !isReadOnlyMode(); if (!analyticsReadOnlyChecked) { analyticsReadOnlyChecked = true; if (isReadOnlyMode()) analyticsEnabled = false; @@ -1383,16 +1384,21 @@ function recordAnalytics(message, worker?) { // Skip writes in read-only mode if (isReadOnlyMode()) return; const report = message.report; - report.metrics = report.metrics.filter( - (metric) => metric.metric !== METRIC.FULL_TEXT || acceptsFullTextTelemetryRow(metric) - ); report.threadId = worker?.threadId || threadId; // Add system information stats as well - for (const metric of report.metrics) { + let retained; + for (let i = 0; i < report.metrics.length; i++) { + const metric = report.metrics[i]; + if (metric.metric === METRIC.FULL_TEXT && !acceptsFullTextTelemetryRow(metric)) { + retained ??= report.metrics.slice(0, i); + continue; + } + retained?.push(metric); if (metric.metric === 'bytes-sent') { totalBytesProcessed += metric.mean * metric.count; } } + if (retained) report.metrics = retained; report.totalBytesProcessed = totalBytesProcessed; if (worker && !isBun) { report.metrics.push({ diff --git a/resources/indexes/fullTextDerivedIndex.ts b/resources/indexes/fullTextDerivedIndex.ts index 73115ba8fe..88fe388385 100644 --- a/resources/indexes/fullTextDerivedIndex.ts +++ b/resources/indexes/fullTextDerivedIndex.ts @@ -794,9 +794,18 @@ export class FullTextDerivedIndexBackend implements DerivedIndexBackend { if (!closeOperation) { fullTextTelemetryCollector.beforeClose(engine); try { - closeOperation = engine.close(options).finally(() => fullTextTelemetryCollector.afterClose(engine)); + closeOperation = engine.close(options).then( + (result) => { + fullTextTelemetryCollector.afterClose(engine); + return result; + }, + (error) => { + fullTextTelemetryCollector.closeFailed(engine); + throw error; + } + ); } catch (error) { - fullTextTelemetryCollector.afterClose(engine); + fullTextTelemetryCollector.closeFailed(engine); throw error; } this.#closeOperations.set(engine, closeOperation); diff --git a/unitTests/config/fullTextTelemetryConfiguration.test.js b/unitTests/config/fullTextTelemetryConfiguration.test.js index 511bcdb5dc..4c5cd723bb 100644 --- a/unitTests/config/fullTextTelemetryConfiguration.test.js +++ b/unitTests/config/fullTextTelemetryConfiguration.test.js @@ -13,12 +13,12 @@ const { waitFor } = require('../waitFor'); const PARAM = 'analytics_fullText'; const root = (fullText) => ({ analytics: { fullText } }); -// Every observer here uses a real file and shared-watcher implementation; no native binding is loaded. describe('Live full-text telemetry configuration', function () { this.timeout(20_000); let directory, file, watcher, stop, saved, savedOverrides, savedEnv; beforeEach(() => { - saved = structuredClone(env.get(PARAM)); + directory = saved = savedOverrides = savedEnv = undefined; + saved = { value: structuredClone(env.get(PARAM)) }; savedOverrides = Object.entries(env.getConfigOverrides() ?? {}).filter( ([key]) => key === PARAM || key.startsWith(PARAM + '_') ); @@ -33,14 +33,16 @@ describe('Live full-text telemetry configuration', function () { stop?.(); watcher?.close(); stop = watcher = undefined; - for (const [key, value] of Object.entries(savedEnv)) { + for (const [key, value] of Object.entries(savedEnv ?? {})) { if (value === undefined) delete process.env[key]; else process.env[key] = value; } - env.refreshConfigSubtree(PARAM, root({ enabled: Symbol('previous') }), {}, resolveFullTextTelemetryPolicy); - updateConfigObject(PARAM, saved); - for (const [name, value] of savedOverrides) env.setProperty(name, value); - rmSync(directory, { recursive: true, force: true }); + if (savedOverrides) { + env.refreshConfigSubtree(PARAM, root({ enabled: Symbol('previous') }), {}, resolveFullTextTelemetryPolicy); + } + if (saved) updateConfigObject(PARAM, saved.value); + for (const [name, value] of savedOverrides ?? []) env.setProperty(name, value); + if (directory) rmSync(directory, { recursive: true, force: true }); }); function observe(value, samples) { writeFileSync(file, stringify(root(value))); @@ -100,6 +102,12 @@ describe('Live full-text telemetry configuration', function () { writeFileSync(file, stringify(root({ enabled: false, intervalMilliseconds: 3000 }))); await waitFor(() => watcher.config?.analytics.fullText.intervalMilliseconds === 3000); assert.strictEqual(samples.at(-1).intervalMilliseconds, 2000); + let unavailable = false; + watcher.once('unavailable', () => (unavailable = true)); + rmSync(file); + await waitFor(() => unavailable); + assert.strictEqual(samples.at(-1).enabled, true); + assert.strictEqual(samples.at(-1).intervalMilliseconds, 2000); }); for (const override of [ [PARAM, { enabled: true, intervalMilliseconds: 2000 }], diff --git a/unitTests/resources/fixtures/fullTextTelemetryWorker.cjs b/unitTests/resources/fixtures/fullTextTelemetryWorker.cjs new file mode 100644 index 0000000000..e5adacb590 --- /dev/null +++ b/unitTests/resources/fixtures/fullTextTelemetryWorker.cjs @@ -0,0 +1,92 @@ +'use strict'; +const { parentPort, threadId, workerData } = require('node:worker_threads'); +workerData.noServerStart = true; +require('#src/server/threads/manageThreads'); +const env = require('#src/utility/environment/environmentManager'); +env.initSync(); +const { loadFullTextNativeBinding } = require('#src/resources/indexes/fullTextNativeBinding'); +const analytics = require('#src/resources/analytics/write'); +const { + initializeFullTextTelemetry, + registerNativeFullTextTelemetry, + subscribeFullTextTelemetry, + fullTextTelemetryCollector, +} = require('#src/resources/analytics/fullTextTelemetryRuntime'); +const { waitFor } = require('../../waitFor'); +let writer; +let removeConsumer; +let barrier; +let pendingFailure; + +parentPort.on('message', ({ id, command, ...options }) => { + if (!command) return; + void (async () => { + if (command === 'produce') { + env.setProperty('analytics_fullText', { enabled: true, intervalMilliseconds: 60_000 }); + initializeFullTextTelemetry(); + analytics.setAnalyticsEnabled(true); + await waitFor(() => fullTextTelemetryCollector.collecting, 5000); + const binding = await loadFullTextNativeBinding(); + writer = await binding.openNativeFullTextIndex({ + path: options.path, + indexId: options.label, + generation: 'generation-1', + fields: [{ name: 'title' }], + analyzer: 'english@2', + positions: true, + limits: { + indexingThreads: 2, + searchThreads: 2, + writerMemoryBytes: 64 * 1024 * 1024, + maxQueuedCommands: 16, + maxQueuedBytes: 64 * 1024 * 1024, + maxBatchBytes: 8 * 1024 * 1024, + }, + }); + registerNativeFullTextTelemetry(writer, options.label, 'writer'); + await writer.applyMutationBatch({ + upserts: [{ id: 'widget', fields: { title: 'catalog widget' } }], + deletes: [], + }); + await writer.publish('worker-telemetry'); + barrier = new Int32Array(options.barrier); + let paused = false; + removeConsumer = subscribeFullTextTelemetry( + (sample) => { + if (paused || sample.index !== options.label || !sample.snapshot?.available) return; + paused = true; + fullTextTelemetryCollector.beforeClose(writer); + parentPort.postMessage({ + event: 'sample-before-main-policy', + sampledAt: sample.sampledAt, + threadId, + completed: sample.snapshot.native.operations.applyFrame.completed.toString(), + }); + // Hold only this synthetic worker's timer turn until main has applied its operator policy. + if (Atomics.wait(barrier, 0, 0, 10_000) === 'timed-out') + pendingFailure = new Error('Main did not release telemetry report ordering barrier'); + }, + { intervalMilliseconds: 60_000 } + ); + fullTextTelemetryCollector.collect(writer); + if (pendingFailure) throw pendingFailure; + return { paused: true, threadId }; + } + if (command === 'close') { + if (writer) { + fullTextTelemetryCollector.beforeClose(writer); + await writer.close({ mode: 'rollback' }); + fullTextTelemetryCollector.afterClose(writer); + writer = undefined; + } + removeConsumer?.(); + fullTextTelemetryCollector.stop(); + return { closed: true }; + } + throw new Error(`Unknown telemetry worker command '${command}'`); + })().then( + (result) => parentPort.postMessage({ id, result }), + (error) => parentPort.postMessage({ id, error: { message: error.message, code: error.code } }) + ); +}); +parentPort.ref(); diff --git a/unitTests/resources/nativeFullTextTelemetry.test.js b/unitTests/resources/nativeFullTextTelemetry.test.js index 1dbda3602e..89c577d375 100644 --- a/unitTests/resources/nativeFullTextTelemetry.test.js +++ b/unitTests/resources/nativeFullTextTelemetry.test.js @@ -35,12 +35,12 @@ describe('Published full-text telemetry collection', function () { binding = await loadFullTextNativeBinding(); }); beforeEach(() => { - collector = new FullTextTelemetryCollector(); handles = []; + collector = new FullTextTelemetryCollector(); }); afterEach(async () => { - collector.stop(); - for (const handle of handles.reverse()) await handle.close({ mode: 'rollback' }); + collector?.stop(); + for (const handle of handles?.reverse() ?? []) await handle.close({ mode: 'rollback' }); }); async function writer() { const options = { @@ -108,6 +108,16 @@ describe('Published full-text telemetry collection', function () { assert.strictEqual(resumed.native_snapshotBytesEnabled, true); assert.strictEqual(resumed.native_reader_snapshotBytesPending, true); assert.doesNotThrow(() => JSON.stringify(resumed)); + collector.setOperatorPolicy({ enabled: true, histograms: true }); + await handle.search({ text: 'widget' }); + collector.collect(handle); + const recordingAgain = row(); + assert.strictEqual(recordingAgain.series, first.series); + assert.strictEqual(recordingAgain.native_operations_search_completed, '4'); + assert.strictEqual( + recordingAgain.native_operations_search_execution_histogram_count, + (histogramCount + 1n).toString() + ); }); it('unions removable consumer options without persisting consumer-only observations', async () => { const { index } = await writer(); @@ -162,6 +172,11 @@ describe('Published full-text telemetry collection', function () { assert.strictEqual(before.native_operations_search_completed, '2'); assert.strictEqual(before.native_reader_documents, '1'); collector.beforeClose(first); + assert.deepStrictEqual( + collector.takeRows(), + [], + 'pausing a retiring peer must not publish a false unavailable row' + ); collector.collect(first); await first.close(); collector.afterClose(first); @@ -182,6 +197,36 @@ describe('Published full-text telemetry collection', function () { assert.strictEqual(current.series, before.series); assert.strictEqual(current.native_operations_search_completed, '4'); }); + it('starts a usable new series after reader retirement lost terminal coverage while collection was off', async () => { + const { options } = await writer(); + const first = await reader(options); + const second = await reader(options); + collector.register(first, 'catalog.Product/search', 'reader'); + collector.register(second, 'catalog.Product/search', 'reader'); + collector.setOperatorPolicy({ enabled: true }); + await first.search({ text: 'widget' }); + await second.search({ text: 'widget' }); + collector.collect(first); + collector.collect(second); + const before = row(); + collector.setOperatorPolicy({ enabled: false }); + collector.beforeClose(first); + await first.close(); + collector.afterClose(first); + collector.setOperatorPolicy({ enabled: true }); + collector.collect(second); + const resumed = row(); + assert.strictEqual(resumed.complete, true); + assert.notStrictEqual(resumed.series, before.series); + await second.search({ text: 'widget' }); + collector.collect(second); + const next = row(); + assert.strictEqual(next.series, resumed.series); + assert.strictEqual( + BigInt(next.native_operations_search_completed) - BigInt(resumed.native_operations_search_completed), + 1n + ); + }); it('contains rejected asynchronous consumers with one callback in flight', async () => { const { index } = await writer(); collector.register(index, 'catalog.Product/search', 'writer'); @@ -205,6 +250,24 @@ describe('Published full-text telemetry collection', function () { reject(new Error('removed consumer failed')); await Promise.resolve(); }); + it('releases the consumer slot after a promise-like callback without catch or finally methods', async () => { + const { index } = await writer(); + collector.register(index, 'catalog.Product/search', 'writer'); + let calls = 0; + const remove = collector.subscribe(() => { + calls++; + return { then: (resolve) => queueMicrotask(resolve) }; + }); + try { + collector.collect(index); + await waitFor(() => { + collector.collect(index); + return calls === 2; + }); + } finally { + remove(); + } + }); it('contains a real closed-handle configuration failure without poisoning other handles', async () => { const { index } = await writer(); await index.close(); @@ -231,6 +294,9 @@ describe('Published full-text telemetry collection', function () { const rows = collector.takeRows(); assert.strictEqual(rows.length, 2); assert.strictEqual(rows.at(-1).droppedRows, '1'); + collector.collect(handles[0]); + const recovered = collector.takeRows().find((value) => value.path === 'catalog.Product0/search'); + assert.strictEqual(recovered.complete, true, 'dropping a pending observation does not lose cumulative counters'); }); it('uses exact uint64 deltas above safe integer range and across wrap', () => { const previous = (1n << 63n) + 1n; @@ -280,15 +346,20 @@ describe('Harper full-text analytics lifecycle and persistence', function () { const analyticsRead = require('#src/resources/analytics/read'); const { table, databases } = require('#src/resources/databases'); const { fullTextDerivedIndexReadiness } = require('#src/resources/derivedIndexes'); - const { setMainIsWorker } = require('#src/server/threads/manageThreads'); + const { setMainIsWorker, getWorkerIndex } = require('#src/server/threads/manageThreads'); + const { server } = require('#src/server/Server'); const { getConfigFilePath } = require('#src/config/configUtils'); const { fullTextTelemetryCollector: live } = require('#src/resources/analytics/fullTextTelemetryRuntime'); setupTestDBPath(); - setMainIsWorker(true); const file = getConfigFilePath(); const original = fs.readFileSync(file, 'utf8'); const config = YAML.parse(original); const wasEnabled = analytics.checkAnalyticsEnabled(); + const wasMainWorker = getWorkerIndex() !== undefined; + const originalGlobals = ['databases', 'server'].map((name) => [ + name, + Object.getOwnPropertyDescriptor(global, name), + ]); let Product; const label = `telemetry-live-${Date.now()}`; function change(enabled) { @@ -326,6 +397,9 @@ describe('Harper full-text analytics lifecycle and persistence', function () { return found.sort(); } try { + setMainIsWorker(true); + global.databases = databases; + global.server = server; analytics.setAnalyticsEnabled(true); Product = table({ database: label, @@ -395,13 +469,24 @@ describe('Harper full-text analytics lifecycle and persistence', function () { 10000 ); } finally { - fs.writeFileSync(file, original); - await waitFor(() => !live.collecting, 5000); - if (Product) { - await Product.fullTextQueryIndexes.catalogSearch.customIndex.close(); - await Product.derivedIndexRuntime?.close(); + try { + fs.writeFileSync(file, original); + await waitFor(() => !live.collecting, 5000); + } finally { + try { + if (Product) { + await Product.fullTextQueryIndexes.catalogSearch.customIndex.close(); + await Product.derivedIndexRuntime?.close(); + } + } finally { + analytics.setAnalyticsEnabled(wasEnabled); + setMainIsWorker(wasMainWorker); + for (const [name, descriptor] of originalGlobals) { + if (descriptor) Object.defineProperty(global, name, descriptor); + else delete global[name]; + } + } } - analytics.setAnalyticsEnabled(wasEnabled); } }); }); diff --git a/unitTests/resources/nativeFullTextTelemetryClose.test.js b/unitTests/resources/nativeFullTextTelemetryClose.test.js new file mode 100644 index 0000000000..67008b6ae0 --- /dev/null +++ b/unitTests/resources/nativeFullTextTelemetryClose.test.js @@ -0,0 +1,116 @@ +require('../testUtils'); +const assert = require('node:assert'); +const { join } = require('node:path'); +const { setupTestDBPath } = require('../testUtils'); +const { waitFor } = require('../waitFor'); +const { supportsPublishedFullTextBinding } = require('./fullTextNativePlatform'); +const { loadFullTextNativeBinding } = require('#src/resources/indexes/fullTextNativeBinding'); +const { NativeFullTextDerivedIndexLifecycle } = require('#src/resources/indexes/nativeFullTextDerivedIndexLifecycle'); +const { + FullTextDerivedIndexBackend, + encodeFullTextCursorPayload, +} = require('#src/resources/indexes/fullTextDerivedIndex'); +const { DERIVED_INDEX_ACCEPTED } = require('#src/resources/derivedIndexRuntime'); +const { subscribeFullTextTelemetry } = require('#src/resources/analytics/fullTextTelemetryRuntime'); + +it('keeps observing an installed real writer after a rejected require-clean close and closes on retry', async function () { + this.timeout(20_000); + if (!(await supportsPublishedFullTextBinding())) this.skip(); + const binding = await loadFullTextNativeBinding(); + const id = `close-retry-${process.pid}`; + const lifecycle = new NativeFullTextDerivedIndexLifecycle({ + storePath: join(setupTestDBPath(), id), + storeName: 'catalog.Product.title', + indexId: id, + sourceGeneration: 'generation-1', + fields: [{ name: 'title' }], + analyzer: 'english@2', + positions: true, + limits: { + indexingThreads: 2, + searchThreads: 2, + writerMemoryBytes: 64 * 1024 * 1024, + maxQueuedCommands: 16, + maxQueuedBytes: 64 * 1024 * 1024, + maxBatchBytes: 8 * 1024 * 1024, + }, + binding, + }); + await lifecycle.initialize({ reclaimRetired: false }); + let writer; + let latest; + const removeConsumer = subscribeFullTextTelemetry( + (sample) => { + if (sample.index === id && sample.role === 'writer' && sample.snapshot?.available) latest = sample; + }, + { intervalMilliseconds: 1000 } + ); + const backend = new FullTextDerivedIndexBackend({ + id, + lifecycle: { + inspect: () => lifecycle.inspect(), + reset: (epoch) => lifecycle.reset(epoch), + async open() { + writer = await lifecycle.open(); + return writer; + }, + }, + closeTimeoutMilliseconds: 5000, + shutdownTimeoutMilliseconds: 10_000, + }); + backend.attach({ + isOwnerEpoch: (epoch) => epoch === 1n, + getReadiness: () => ({ state: 'ready', ownerEpoch: 1n, rebuildAttempts: 0 }), + publicationChanged() {}, + }); + const cursor = { format: 1, logs: { local: 1 } }; + try { + backend.getDurableCursor(); + assert.strictEqual( + backend.deliver({ + ownerEpoch: 1n, + transactions: [], + records: [ + { + tableId: 1, + recordId: 'seed', + logVersion: 1, + state: { kind: 'record', version: 1, projection: { title: 'catalog widget' } }, + }, + ], + through: cursor, + bytes: 64, + }), + DERIVED_INDEX_ACCEPTED + ); + backend.flush(); + await waitFor(() => backend.getDurableCursor()?.logs.local === 1, 5000); + await waitFor(() => latest?.snapshot.native.operations.applyFrame.completed >= 1n, 3000); + const series = latest.series; + // Stage real native work outside the backend so its clean-close bookkeeping cannot mask E_DIRTY_CLOSE. + await writer.applyMutationBatch({ + upserts: [{ id: 'external', fields: { title: 'catalog widget' } }], + deletes: [], + }); + assert.ok(writer.status().uncommittedMutations > 0n); + await assert.rejects(backend.shutdown(1n), /did not prove quiescence/); + assert.strictEqual(writer.status().state, 'open'); + assert.strictEqual(writer.telemetry().native.operations.close.errors.E_DIRTY_CLOSE, 1n); + const before = writer.telemetry().native.operations.applyFrame.completed; + await writer.applyMutationBatch({ upserts: [{ id: 'later', fields: { title: 'catalog widget' } }], deletes: [] }); + await waitFor(() => latest?.snapshot.native.operations.applyFrame.completed > before, 3000); + assert.strictEqual(latest.series, series); + await writer.publish(encodeFullTextCursorPayload(cursor)); + await backend.shutdown(1n); + assert.strictEqual(writer.status().state, 'closed'); + assert.strictEqual(latest.snapshot.native.state, 'closed'); + assert.strictEqual(latest.snapshot.native.operations.close.completed, 2n); + } finally { + try { + if (writer && writer.status().state !== 'closed') await writer.close({ mode: 'rollback' }); + await backend.shutdown(1n); + } finally { + removeConsumer(); + } + } +}); diff --git a/unitTests/resources/nativeFullTextTelemetryWorkers.test.js b/unitTests/resources/nativeFullTextTelemetryWorkers.test.js new file mode 100644 index 0000000000..2e072cb4f0 --- /dev/null +++ b/unitTests/resources/nativeFullTextTelemetryWorkers.test.js @@ -0,0 +1,152 @@ +require('../testUtils'); +const assert = require('node:assert'); +const { readFileSync, writeFileSync } = require('node:fs'); +const { join } = require('node:path'); +const { setupTestDBPath, ensureSystemTables } = require('../testUtils'); +const { setRootConfigEntry, readRootConfig } = require('../rootConfigFixture'); +const { waitFor } = require('../waitFor'); +const { getConfigFilePath } = require('#src/config/configUtils'); +const { getSharedRootConfigWatcher } = require('#src/config/RootConfigWatcher'); +const { getDatabases } = require('#src/resources/databases'); +const { startWorker } = require('#src/server/threads/manageThreads'); +const { supportsPublishedFullTextBinding } = require('./fullTextNativePlatform'); +const { loadFullTextNativeBinding } = require('#src/resources/indexes/fullTextNativeBinding'); +const { + initializeFullTextTelemetry, + fullTextTelemetryCollector, +} = require('#src/resources/analytics/fullTextTelemetryRuntime'); +const analytics = require('#src/resources/analytics/write'); + +it('persists an actual managed-worker sample produced before main applies the same enabled policy', async function () { + this.timeout(30_000); + if (process.env.HARPER_STORAGE_ENGINE === 'lmdb' || !(await supportsPublishedFullTextBinding())) this.skip(); + const directory = setupTestDBPath(); + await ensureSystemTables(); + await loadFullTextNativeBinding(); + const configPath = getConfigFilePath(); + const savedConfig = readFileSync(configPath, 'utf8'); + const wasEnabled = analytics.checkAnalyticsEnabled(); + const barrier = new Int32Array(new SharedArrayBuffer(4)); + const label = `worker-telemetry-${process.pid}`; + const pending = new Map(); + let sequence = 0; + let worker; + let produced; + let producer; + function request(command, options = {}) { + return new Promise((resolve, reject) => { + const id = ++sequence; + const timer = setTimeout(() => { + pending.delete(id); + reject(new Error(`Telemetry worker command ${command} did not settle`)); + }, 15_000); + pending.set(id, { resolve, reject, timer }); + try { + worker.postMessage({ id, command, ...options }); + } catch (error) { + clearTimeout(timer); + pending.delete(id); + reject(error); + } + }); + } + try { + initializeFullTextTelemetry(); + analytics.setAnalyticsEnabled(true); + setRootConfigEntry('analytics', { ...readRootConfig().analytics, fullText: { enabled: false } }); + const watcher = getSharedRootConfigWatcher(); + await watcher.ready; + await waitFor( + () => watcher.config?.analytics?.fullText?.enabled === false && !fullTextTelemetryCollector.collecting, + 5000 + ); + worker = startWorker(join(__dirname, 'fixtures/fullTextTelemetryWorker.cjs'), { + name: 'http', + workerIndex: 1, + autoRestart: false, + }); + worker.on('message', (message) => { + if (message.event === 'sample-before-main-policy') produced = message; + const command = pending.get(message.id); + if (!command) return; + clearTimeout(command.timer); + pending.delete(message.id); + if (message.error) command.reject(Object.assign(new Error(message.error.message), message.error)); + else command.resolve(message.result); + }); + worker.on('error', (error) => { + for (const command of pending.values()) { + clearTimeout(command.timer); + command.reject(error); + } + pending.clear(); + }); + worker.on('exit', (code) => { + for (const command of pending.values()) { + clearTimeout(command.timer); + command.reject(new Error(`Telemetry worker exited before its command settled: ${code}`)); + } + pending.clear(); + }); + producer = request('produce', { path: join(directory, label), label, barrier: barrier.buffer }); + // Attach rejection immediately while the ordering assertion waits for the producer's first event. + void producer.catch(() => {}); + await waitFor(() => produced, 10_000); + assert.ok(BigInt(produced.completed) > 0n); + assert.strictEqual(fullTextTelemetryCollector.collecting, false); + await waitFor(() => Date.now() > produced.sampledAt); + setRootConfigEntry('analytics', { ...readRootConfig().analytics, fullText: { enabled: true } }); + await waitFor(() => fullTextTelemetryCollector.collecting, 5000); + const mainEnabledAt = Date.now(); + assert.ok(produced.sampledAt < mainEnabledAt); + Atomics.store(barrier, 0, 1); + Atomics.notify(barrier, 0); + await producer; + const findRow = () => { + const store = getDatabases().system?.hdb_raw_analytics?.primaryStore; + if (!store) return; + for (const { value } of store.getRange({ start: false, end: Infinity })) { + if (value.threadId !== worker.threadId) continue; + const row = value.metrics.find( + (entry) => entry.metric === 'fulltext' && entry.path === label && entry.sampledAt === produced.sampledAt + ); + if (row) return { report: value, row }; + } + }; + await waitFor(findRow, 5000); + const { report, row } = findRow(); + assert.strictEqual(report.threadId, produced.threadId); + assert.strictEqual(row.method, 'writer'); + assert.strictEqual(row.available, true); + assert.ok(BigInt(row.native_operations_applyFrame_completed) > 0n); + assert.ok(row.sampledAt < mainEnabledAt); + assert.deepStrictEqual(await request('close'), { closed: true }); + } finally { + Atomics.store(barrier, 0, 1); + Atomics.notify(barrier, 0); + await producer?.catch(() => {}); + try { + if (worker) { + try { + if (worker.threadId !== -1) await request('close'); + } finally { + let timer; + try { + await Promise.race([ + worker.terminate(), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('Telemetry worker termination did not settle')), 5000); + }), + ]); + } finally { + clearTimeout(timer); + } + } + } + } finally { + for (const command of pending.values()) clearTimeout(command.timer); + writeFileSync(configPath, savedConfig); + analytics.setAnalyticsEnabled(wasEnabled); + } + } +}); From 1d7bccb960e85332856ed24a723c6fd59540bb2f Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Mon, 2 Nov 2026 22:38:57 -0700 Subject: [PATCH 03/26] Keep full-text telemetry identities distinct and contain lifecycle failures --- benchmarks/fulltext-analytics.bench.js | 22 +++++++-- benchmarks/fulltext-analytics.mjs | 12 +++-- resources/FULL_TEXT.md | 2 +- resources/analytics/fullTextTelemetry.ts | 13 +++++- .../analytics/fullTextTelemetryRuntime.ts | 35 +++++++++------ resources/derivedIndexes.ts | 6 ++- resources/indexes/fullTextQueryIndex.ts | 11 ++++- .../fullTextTelemetryConfiguration.test.js | 21 +++++++++ .../resources/nativeFullTextTelemetry.test.js | 45 +++++++++++++++++++ validation/configValidator.ts | 4 +- 10 files changed, 146 insertions(+), 25 deletions(-) diff --git a/benchmarks/fulltext-analytics.bench.js b/benchmarks/fulltext-analytics.bench.js index 719981712b..bab49519f4 100644 --- a/benchmarks/fulltext-analytics.bench.js +++ b/benchmarks/fulltext-analytics.bench.js @@ -82,8 +82,8 @@ it(`qualifies actual Harper fulltext analytics (${configuration.profile})`, asyn attribute: 'catalogSearch', comparator: 'matches', value: 'widget', - maxIndexLagMilliseconds: 0, - waitForIndexMilliseconds: 5000, + maxIndexLagMilliseconds: configuration.maxIndexLag, + ...(configuration.maxIndexLag === 0 ? { waitForIndexMilliseconds: 5000 } : null), }, ], limit: 4, @@ -214,6 +214,8 @@ it(`qualifies actual Harper fulltext analytics (${configuration.profile})`, asyn writerDelayMilliseconds: 50, configuredDurationMilliseconds: configuration.duration, configuredIntervalMilliseconds: configuration.interval, + maxIndexLagMilliseconds: configuration.maxIndexLag, + waitForIndexMilliseconds: configuration.maxIndexLag === 0 ? 5000 : null, histograms: false, snapshotBytes: false, startupWarmupMilliseconds: 2000, @@ -247,7 +249,21 @@ it(`qualifies actual Harper fulltext analytics (${configuration.profile})`, asyn } catch (error) { writeFileSync( join(configuration.output, 'failure.json'), - JSON.stringify({ name: error.name, message: error.message, stack: error.stack }, null, 2) + JSON.stringify( + { + name: error.name, + message: error.message, + stack: error.stack, + causes: + error instanceof AggregateError + ? error.errors + .slice(0, 8) + .map((cause) => ({ name: cause.name, message: cause.message, stack: cause.stack })) + : undefined, + }, + null, + 2 + ) ); throw error; } finally { diff --git a/benchmarks/fulltext-analytics.mjs b/benchmarks/fulltext-analytics.mjs index 295fa33087..fb0b1fc00e 100644 --- a/benchmarks/fulltext-analytics.mjs +++ b/benchmarks/fulltext-analytics.mjs @@ -5,15 +5,21 @@ import { fileURLToPath } from 'node:url'; import { createHash } from 'node:crypto'; const repository = resolve(dirname(fileURLToPath(import.meta.url)), '..'); -const options = { duration: 70_000, indexes: 4, pairs: 2, interval: 60_000, output: undefined }; -const bounds = { duration: [100, 3_600_000], indexes: [1, 16], pairs: [1, 10], interval: [1000, 2_147_483_647] }; +const options = { duration: 70_000, indexes: 4, pairs: 2, interval: 60_000, maxIndexLag: 0, output: undefined }; +const bounds = { + duration: [100, 3_600_000], + indexes: [1, 16], + pairs: [1, 10], + interval: [1000, 2_147_483_647], + maxIndexLag: [0, 60_000], +}; for (let i = 2; i < process.argv.length; i += 2) { if (!process.argv[i].startsWith('--')) throw new Error('Benchmark options must use --name value'); const name = process.argv[i].slice(2); const value = process.argv[i + 1]; if (!Object.hasOwn(options, name) || value === undefined) throw new Error( - 'Use --duration milliseconds --indexes 1..16 --pairs 1..10 --interval milliseconds --output directory' + 'Use --duration milliseconds --indexes 1..16 --pairs 1..10 --interval milliseconds --maxIndexLag milliseconds --output directory' ); if (name === 'output') options.output = resolve(value); else { diff --git a/resources/FULL_TEXT.md b/resources/FULL_TEXT.md index 3115e64edf..1a90d413e2 100644 --- a/resources/FULL_TEXT.md +++ b/resources/FULL_TEXT.md @@ -211,7 +211,7 @@ Set `enabled: false` or remove the block to stop scheduled polling and persisten `histograms: true` and `snapshotBytes: true` independently enable optional recording/reader-byte observation; both default to false. Turning collection off disables options no remaining consumer needs on live handles; a handle already closing keeps its last options until close settles. Re-enabling resumes cumulative counters. Enabling bytes schedules no native publication: values can remain pending until a successful publication/reload opportunity. Cached bytes, observation age and errors retain their own meanings; reader bytes are not directory totals or resident memory. -Read `get_analytics` with `metric: fulltext`. Each stored row identifies `path` (`.
/`, or `index:` when the label exceeds 256 bytes), `method` (`writer` or `reader`), worker `thread`, a bounded-lifetime `series` reset marker and actual `sampledAt`. Field names flatten the wrapper snapshot: for example, `native_operations_search_completed`, `native_operations_search_queue_sumNanoseconds`, `native_queues_ordinarySearch_commands` and `logicalBatches_completed`. Values are flat JSON scalars; exact bigint counters, nanoseconds, bytes, histogram counts and bounds are decimal strings. Availability/pending/error flags distinguish missing observations from zero. Null can mean an unavailable observation, an empty queue's oldest age, or a histogram's unbounded final bucket. Histograms retain noncumulative buckets; no percentiles are invented. Histogram deltas cover only intervals with recording enabled, not an entire collection gap when recording was disabled. +Read `get_analytics` with `metric: fulltext`. Each stored row identifies `path` (`.
/` with percent-encoded components, including literal dots as `%2E`, or `index:` when a component cannot be percent-encoded or the encoded label exceeds 256 bytes), `method` (`writer` or `reader`), worker `thread`, a bounded-lifetime `series` reset marker and actual `sampledAt`. Field names flatten the wrapper snapshot: for example, `native_operations_search_completed`, `native_operations_search_queue_sumNanoseconds`, `native_queues_ordinarySearch_commands` and `logicalBatches_completed`. Values are flat JSON scalars; exact bigint counters, nanoseconds, bytes, histogram counts and bounds are decimal strings. Availability/pending/error flags distinguish missing observations from zero. Null can mean an unavailable observation, an empty queue's oldest age, or a histogram's unbounded final bucket. Histograms retain noncumulative buckets; no percentiles are invented. Histogram deltas cover only intervals with recording enabled, not an entire collection gap when recording was disabled. Native uint64 counter wraps are unwrapped into exact logical-series totals. Reader reopen and overlapping retired readers carry counters into the same worker/index/role series; document/segment/generation observations come from the most recently observed eligible reader rather than summing duplicate snapshots. Queue occupancy sums live handles of the same role; oldest age uses their maximum. Shared queue capacities overlap: never add ordinary/expensive capacities or treat them as a process-wide budget. Final close counters are sampled once after settlement; no recurring scrape runs while close is in flight. diff --git a/resources/analytics/fullTextTelemetry.ts b/resources/analytics/fullTextTelemetry.ts index cd5afe6349..d9d7551b38 100644 --- a/resources/analytics/fullTextTelemetry.ts +++ b/resources/analytics/fullTextTelemetry.ts @@ -1,4 +1,4 @@ -import { randomUUID } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import type { ConfigureTelemetryOptions, FullTextTelemetry } from '@harperfast/fulltext/native'; import { METRIC } from './metadata.ts'; @@ -116,6 +116,17 @@ export function flattenFullTextTelemetry(snapshot: FullTextTelemetry): FlatSnaps return { counters, gauges }; } +export function fullTextTelemetryIndexLabel(database: string, table: string, index: string): string { + const components = [database, table, index]; + const encode = (value: string) => encodeURIComponent(value).replaceAll('.', '%2E'); + try { + return `${encode(database)}.${encode(table)}/${encode(index)}`; + } catch (error) { + if (!(error instanceof URIError)) throw error; + return `index:${createHash('sha256').update(JSON.stringify(components)).digest('hex')}`; + } +} + export class FullTextTelemetryCollector { #groups = new Map(); #entries = new Set(); diff --git a/resources/analytics/fullTextTelemetryRuntime.ts b/resources/analytics/fullTextTelemetryRuntime.ts index 67d9085f5c..9328f1b340 100644 --- a/resources/analytics/fullTextTelemetryRuntime.ts +++ b/resources/analytics/fullTextTelemetryRuntime.ts @@ -20,15 +20,19 @@ let removeListener: (() => void) | undefined; let persistenceActive = false; let nextWarningAt = 0; +function reportFailure() { + if (Date.now() < nextWarningAt) return; + nextWarningAt = Date.now() + 60_000; + try { + logger.warn?.('Could not configure, collect or deliver full-text telemetry'); + } catch {} +} + export const fullTextTelemetryCollector = new FullTextTelemetryCollector({ onPending: () => { if (operatorPolicy.enabled && checkAnalyticsEnabled(false)) requestAnalyticsFlush(); }, - onError: () => { - if (Date.now() < nextWarningAt) return; - nextWarningAt = Date.now() + 60_000; - logger.warn?.('A full-text telemetry observation or consumer failed'); - }, + onError: reportFailure, }); export function setFullTextAnalyticsEnabled(enabled: boolean): void { @@ -48,17 +52,22 @@ export function acceptsFullTextTelemetryRow(row: Record): boole export function initializeFullTextTelemetry(): void { if (started) return; - started = true; - removeListener = addAnalyticsListener((metrics) => { + const listener = addAnalyticsListener((metrics) => { if (!persistenceActive || !checkAnalyticsEnabled(false)) return; const rows = fullTextTelemetryCollector.takeRows(); metrics.push(...rows); }); - stopWatching = watchFullTextTelemetryConfiguration((value) => { - const next = resolveFullTextTelemetryPolicy(value); - operatorPolicy = next; - setFullTextAnalyticsEnabled(checkAnalyticsEnabled(false)); - }); + try { + stopWatching = watchFullTextTelemetryConfiguration((value) => { + operatorPolicy = resolveFullTextTelemetryPolicy(value); + setFullTextAnalyticsEnabled(checkAnalyticsEnabled(false)); + }); + removeListener = listener; + started = true; + } catch { + listener(); + reportFailure(); + } } export function subscribeFullTextTelemetry( @@ -77,7 +86,7 @@ export function registerNativeFullTextTelemetry(handle: unknown, index: string, const label = Buffer.byteLength(index) <= 256 ? index : `index:${createHash('sha256').update(index).digest('hex')}`; fullTextTelemetryCollector.register(native, label, role); } catch { - logger.warn?.('Could not register a full-text telemetry handle'); + reportFailure(); } } diff --git a/resources/derivedIndexes.ts b/resources/derivedIndexes.ts index c8e9c71c86..a4f4781614 100644 --- a/resources/derivedIndexes.ts +++ b/resources/derivedIndexes.ts @@ -1,4 +1,5 @@ import { randomUUID } from 'node:crypto'; +import { fullTextTelemetryIndexLabel } from './analytics/fullTextTelemetry.ts'; import { ClientError } from '../utility/errors/hdbError.ts'; import { settleBeforeDeadline } from '../utility/when.ts'; import { ITC_EVENT_TYPES } from '../utility/hdbTerms.ts'; @@ -403,6 +404,7 @@ export function attachDerivedIndexes( const generation = Table.fullTextIndexGenerations?.[definition.name]; const id = fullTextDerivedIndexId(Table, definition.name); const readinessId = fullTextDerivedIndexReadinessId(Table, definition.name, generation); + const telemetryLabel = fullTextTelemetryIndexLabel(Table.databaseName, Table.tableName, definition.name); if (typeof generation !== 'string' || generation.length === 0) { publishDerivedIndexReadiness(auditStore, readinessId, 'unavailable', 'backend-failed'); fullTextLogger.error?.( @@ -419,7 +421,7 @@ export function attachDerivedIndexes( indexId: id, storePath: Table.primaryStore.rootStore.path, storeName: `${Table.tableName}/${definition.name}`, - telemetryLabel: `${Table.databaseName}.${Table.tableName}/${definition.name}`, + telemetryLabel, sourceGeneration: `${Table.tableId}:${generation}`, limits: { ...FULL_TEXT_READER_LIMITS }, ...(fullTextTest ? { binding: fullTextTest.binding } : null), @@ -479,7 +481,7 @@ export function attachDerivedIndexes( storePath: Table.primaryStore.rootStore.path, storeName, sourceGeneration: `${Table.tableId}:${generation}`, - telemetryLabel: `${Table.databaseName}.${Table.tableName}/${definition.name}`, + telemetryLabel, fields: definition.fields.map(({ name, weight }) => ({ name, weight })), analyzer: storage.analyzer, stopWords: storage.stopWords, diff --git a/resources/indexes/fullTextQueryIndex.ts b/resources/indexes/fullTextQueryIndex.ts index 458086473c..2a5692495c 100644 --- a/resources/indexes/fullTextQueryIndex.ts +++ b/resources/indexes/fullTextQueryIndex.ts @@ -1062,7 +1062,16 @@ export class FullTextQueryIndex { fullTextTelemetryCollector.beforeClose(slot.reader); return slot.reader.close(); }) - .finally(() => fullTextTelemetryCollector.afterClose(slot.reader)) + .then( + (result) => { + fullTextTelemetryCollector.afterClose(slot.reader); + return result; + }, + (error) => { + fullTextTelemetryCollector.closeFailed(slot.reader); + throw error; + } + ) .then((result) => { this.#retiredReaderSlots.delete(slot); if (result?.cleanupError) this.#warnReaderClose(result.cleanupError); diff --git a/unitTests/config/fullTextTelemetryConfiguration.test.js b/unitTests/config/fullTextTelemetryConfiguration.test.js index 4c5cd723bb..f3ce1d6e45 100644 --- a/unitTests/config/fullTextTelemetryConfiguration.test.js +++ b/unitTests/config/fullTextTelemetryConfiguration.test.js @@ -7,6 +7,8 @@ const { watchFullTextTelemetryConfiguration } = require('#src/config/fullTextTel const { resolveFullTextTelemetryPolicy } = require('#src/resources/analytics/fullTextTelemetry'); const env = require('#src/utility/environment/environmentManager'); const { updateConfigObject } = require('#src/config/configUtils'); +const { configValidator } = require('#src/validation/configValidator'); +const { readRootConfig } = require('../rootConfigFixture'); const { setupTestDBPath } = require('../testUtils'); const { waitFor } = require('../waitFor'); @@ -49,6 +51,25 @@ describe('Live full-text telemetry configuration', function () { watcher = new RootConfigWatcher(file); stop = watchFullTextTelemetryConfiguration((policy) => samples.push(policy), watcher); } + it('rejects quoted telemetry values consistently at validation and live application', () => { + for (const value of [ + { enabled: 'true' }, + { histograms: 'false' }, + { snapshotBytes: 'true' }, + { intervalMilliseconds: '1000' }, + ]) { + const config = { ...readRootConfig(), analytics: { fullText: value } }; + assert.match(configValidator(config, true).error?.message, /analytics\.fullText/); + assert.throws(() => resolveFullTextTelemetryPolicy(value)); + } + assert.strictEqual( + configValidator( + { ...readRootConfig(), analytics: { fullText: { enabled: true, intervalMilliseconds: 1000 } } }, + true + ).error, + undefined + ); + }); it('enables, retains a valid policy after an invalid edit, disables and unsubscribes', async () => { const samples = []; observe({ enabled: true, intervalMilliseconds: 1000 }, samples); diff --git a/unitTests/resources/nativeFullTextTelemetry.test.js b/unitTests/resources/nativeFullTextTelemetry.test.js index 89c577d375..f4a997ab06 100644 --- a/unitTests/resources/nativeFullTextTelemetry.test.js +++ b/unitTests/resources/nativeFullTextTelemetry.test.js @@ -11,6 +11,7 @@ const { resolveFullTextTelemetryPolicy, fullTextNativeCounterDelta, flattenFullTextTelemetry, + fullTextTelemetryIndexLabel, } = require('#src/resources/analytics/fullTextTelemetry'); const limits = { @@ -80,6 +81,35 @@ describe('Published full-text telemetry collection', function () { assert.strictEqual((await index.search({ text: 'widget' })).hits.length, 1); assert.strictEqual(index.telemetry().native.histogramsEnabled, false); }); + it('keeps delimiter-bearing index identities in separate series', async () => { + const labels = [ + fullTextTelemetryIndexLabel('a.b', 'c', 'search'), + fullTextTelemetryIndexLabel('a', 'b.c', 'search'), + fullTextTelemetryIndexLabel('a%2Eb', 'c', 'search'), + fullTextTelemetryIndexLabel('a', 'b/c', 'search'), + ]; + assert.strictEqual(new Set(labels).size, labels.length); + assert.strictEqual(fullTextTelemetryIndexLabel('catalog', 'Product', 'search'), 'catalog.Product/search'); + assert.strictEqual(fullTextTelemetryIndexLabel('catalog', 'Product', '😀'), 'catalog.Product/%F0%9F%98%80'); + const illFormed = ['\ud800', '\ud801', '\ufffd'].map((index) => fullTextTelemetryIndexLabel('a', 'b', index)); + assert.strictEqual(new Set(illFormed).size, illFormed.length); + assert.match(illFormed[0], /^index:[a-f0-9]{64}$/); + const { options } = await writer(); + collector.setOperatorPolicy({ enabled: true }); + for (const label of labels) { + const handle = await reader(options); + collector.register(handle, label, 'reader'); + await handle.search({ text: 'widget' }); + collector.collect(handle); + } + const rows = collector.takeRows(); + assert.strictEqual(new Set(rows.map((row) => row.series)).size, labels.length); + assert.deepStrictEqual( + rows.map((row) => row.path), + labels + ); + assert.ok(rows.every((row) => row.native_operations_search_completed === '1')); + }); it('enables, pauses and resumes the same reader with cumulative gap counts and independent options', async () => { const { options } = await writer(); const handle = await reader(options); @@ -468,6 +498,21 @@ describe('Harper full-text analytics lifecycle and persistence', function () { ), 10000 ); + const { subscribeFullTextTelemetry } = require('#src/resources/analytics/fullTextTelemetryRuntime'); + let closing = false; + let closed; + const removeReaderConsumer = subscribeFullTextTelemetry((sample) => { + if (sample.index !== writer.path || sample.role !== 'reader') return; + if (sample.stage === 'closing') closing = true; + if (sample.snapshot?.available && sample.snapshot.native.state === 'closed') closed = sample.snapshot; + }); + try { + await Product.fullTextQueryIndexes.catalogSearch.customIndex.close(); + assert.strictEqual(closing, true); + assert.strictEqual(closed?.native.operations.close.completed, 1n); + } finally { + removeReaderConsumer(); + } } finally { try { fs.writeFileSync(file, original); diff --git a/validation/configValidator.ts b/validation/configValidator.ts index 2e668860b1..bd5649189e 100644 --- a/validation/configValidator.ts +++ b/validation/configValidator.ts @@ -448,7 +448,9 @@ export function configValidator(configJson, skipFsValidation = false) { intervalMilliseconds: number.integer().min(1000).max(2147483647).optional(), histograms: boolean.optional(), snapshotBytes: boolean.optional(), - }).optional(), + }) + .strict() + .optional(), }), replication: Joi.object({ hostname: bareHostConstraints, From 373206fd2d6190ae74f4f081980cf8e6e7feb72b Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Tue, 3 Nov 2026 06:54:25 -0700 Subject: [PATCH 04/26] test: align deployed full-text capabilities with telemetry engine --- integrationTests/resources/full-text-search.test.ts | 5 +++-- resources/FULL_TEXT.md | 2 +- resources/analytics/DESIGN.md | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/integrationTests/resources/full-text-search.test.ts b/integrationTests/resources/full-text-search.test.ts index 4adfdf7664..f109812603 100644 --- a/integrationTests/resources/full-text-search.test.ts +++ b/integrationTests/resources/full-text-search.test.ts @@ -306,8 +306,9 @@ suite('deployed full-text fields and native search', (ctx: ContextWithHarper) => const packageJson = JSON.parse(await readFile(resolve(import.meta.dirname, '../..', 'package.json'), 'utf8')); assert.strictEqual(info.packageVersion, packageJson.optionalDependencies['@harperfast/fulltext']); assert.strictEqual(info.mutationBatchApiVersion, 5); - assert.strictEqual(info.nativeAbiVersion, 13); - assert.strictEqual(info.queryApiVersion, 7); + assert.strictEqual(info.nativeAbiVersion, 14); + assert.strictEqual(info.queryApiVersion, 8); + assert.strictEqual(info.telemetryApiVersion, 1); assert.ok(info.queryClassIsolationMinimumSearchThreads > 0); const description = await operation({ operation: 'describe_table', schema: 'data', table: 'Product' }); const index = description.full_text_indexes.find(({ name }) => name === 'catalogSearch'); diff --git a/resources/FULL_TEXT.md b/resources/FULL_TEXT.md index 1a90d413e2..7c3e26f691 100644 --- a/resources/FULL_TEXT.md +++ b/resources/FULL_TEXT.md @@ -205,7 +205,7 @@ analytics: enabled: true ``` -The implied interval is 60,000 milliseconds per handle. Calls are staggered across existing writer/readers, and their rows use Harper's existing analytics flush, storage, retention and protected `get_analytics` interface. Collection never opens a cold index. To tune it, set `intervalMilliseconds` between 1,000 and 2,147,483,647; synchronous snapshots and export work have measurable cost, so qualify the intended number of indexes and cadence. No production overhead bound is claimed. +The implied periodic interval is 60,000 milliseconds per handle. Calls are staggered across existing writer/readers, and their rows use Harper's existing analytics flush, storage, retention and protected `get_analytics` interface. Successful handle closes also collect final counters, so the interval is not a limit on total snapshot or export frequency. Collection never opens a cold index. To tune it, set `intervalMilliseconds` between 1,000 and 2,147,483,647; synchronous snapshots and export work have measurable cost, so qualify the intended number of indexes, reader churn and cadence. No production overhead bound is claimed. Set `enabled: false` or remove the block to stop scheduled polling and persistent full-text export. A synchronous observation already executing completes. Editing unrelated settings does not restart collection. Invalid telemetry edits retain the last valid policy; deletion of the configuration file pauses file-based operator collection; environment overlays still apply while the file is missing. Disabling analytics also disables its full-text collection demand. Internal registered consumers can independently request observations; removing the last demand stops the timer. Consumer-only observations do not enable persistent analytics. diff --git a/resources/analytics/DESIGN.md b/resources/analytics/DESIGN.md index 4d688c3ab9..60a735e4c0 100644 --- a/resources/analytics/DESIGN.md +++ b/resources/analytics/DESIGN.md @@ -75,6 +75,6 @@ always a caller's measure. `fullTextTelemetry.ts` keeps private per-handle baselines and carries exact counters into a stable worker/index/role series. `nativeFullTextTelemetry.test.js` pins reader churn, wrap/precision, opt-in toggles and final close. The scheduler pauses before native close and observes final scalars only after settlement, so it never polls close in flight. Completed registrations release handle references; inactive groups remain only until their coalesced final row is drained. Pending export has a 128-row memory bound with explicit drop counts. -`fullTextTelemetryRuntime.ts` joins removable internal consumer demand with live operator policy. Only operator demand enables persistent analytics. No demand means no telemetry timer; no cold handles are opened. Counters are decimal strings, rows are flat and `cumulative` makes the aggregation fold retain the newest sample, including availability/error fields, per worker. Lost final handle counts force a new series on the next successful live observation; pending-row drops preserve cumulative totals. Failed writer closes resume collection because Harper may keep the engine installed. Existing request metrics retain their averaging semantics. `aggregationCycle.test.js` verifies exact counters above 2^53 survive storage and `get_analytics`. +`fullTextTelemetryRuntime.ts` joins removable internal consumer demand with live operator policy. Only operator demand enables persistent analytics. No demand means no telemetry timer; no cold handles are opened. Counters are decimal strings, rows are flat and `cumulative` makes the aggregation fold retain the newest sample, including availability/error fields, per worker. Lost final handle counts force a new series on the next successful live observation. Pending-row drops preserve cumulative totals while a group has live handles; dropping its final closed row can lose those totals. Overflow evicts by first pending insertion, and replacing a group's pending observation retains its position. Failed writer closes resume collection because Harper may keep the engine installed. Existing request metrics retain their averaging semantics. `aggregationCycle.test.js` verifies exact counters above 2^53 survive storage and `get_analytics`. `watchFullTextTelemetryConfiguration` uses the shared root watcher, environment overlay and transactional `refreshConfigSubtree`; unrelated edits preserve sampling cadence, invalid edits preserve valid policy, and availability events pause/recover after file deletion. `fullTextTelemetryConfiguration.test.js` names the effective override and recovery controls. Native runtime/query policy remains frozen separately. From 3b18f148aca0e8079a3da01880fb1376c5763777 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Tue, 3 Nov 2026 07:06:06 -0700 Subject: [PATCH 05/26] perf: avoid idle telemetry export work and reuse handle contract --- dependencies.md | 2 +- resources/analytics/fullTextTelemetry.ts | 25 ++++++++++++------- .../analytics/fullTextTelemetryRuntime.ts | 2 +- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/dependencies.md b/dependencies.md index bd33b562bc..d51a471d9d 100644 --- a/dependencies.md +++ b/dependencies.md @@ -290,7 +290,7 @@ This is the inverse of the entries below — a dependency we take deliberate ste ## @harperfast/fulltext (optional dependency) - Need for usage: Supplies the Tantivy-backed native index used by Harper full-text derived indexes. Harper remains the source of truth; each node builds and advances its local index from committed records through the shared derived-index runtime. -- Size/memory cost: The exact-pinned 0.9.0 root package includes JavaScript facades and types plus one platform-specific native package. Runtime memory and native index size depend on the indexed text, analyzer, and stored-field options. +- Size/memory cost: The exact-pinned 0.9.0 root package includes JavaScript facades and types plus one platform-specific native package. A local macOS arm64 install measured with `du -sk` uses 856 KiB for the root and 9,944 KiB for its platform package; these are allocated disk sizes, not resident memory. Runtime memory and native index size depend on the indexed text, analyzer, and stored-field options. - Security: First-party Apache-2.0 Harper package wrapping Tantivy. It runs native code in-process, so Harper exact-pins the root package and the package exact-pins every platform prebuild to the same version. - Environment interaction: Lazily loaded only when a full-text index is activated. It creates Tantivy index files in Harper's derived-index directory; those files are disposable local state and are rebuilt or replayed from Harper records after loss or incompatibility. Upgrading the 0.5.0 engine shipped in Harper 5.3.4 requires a full local rebuild on every node; see the [deployment and recovery guidance](resources/FULL_TEXT.md#upgrading-an-existing-native-index). - Overlap: None. Harper owns schema, transactions, replication, source-record reads, derived-index coordination and opt-in analytics scheduling/export; the package owns native indexing, persistence, search execution and native/facade telemetry accounting. Harper requires native ABI 14, query API 8 and telemetry API 1. diff --git a/resources/analytics/fullTextTelemetry.ts b/resources/analytics/fullTextTelemetry.ts index d9d7551b38..ed9e21c088 100644 --- a/resources/analytics/fullTextTelemetry.ts +++ b/resources/analytics/fullTextTelemetry.ts @@ -1,13 +1,14 @@ import { createHash, randomUUID } from 'node:crypto'; import type { ConfigureTelemetryOptions, FullTextTelemetry } from '@harperfast/fulltext/native'; import { METRIC } from './metadata.ts'; +import type { NativeFullTextTelemetryHandle } from '../indexes/fullTextNativeBinding.ts'; export type FullTextTelemetryPolicy = Required & { enabled: boolean; intervalMilliseconds: number; }; -export const DEFAULT_FULL_TEXT_TELEMETRY_POLICY: Readonly = Object.freeze({ +const DEFAULT_FULL_TEXT_TELEMETRY_POLICY: Readonly = Object.freeze({ enabled: false, intervalMilliseconds: 60_000, histograms: false, @@ -34,10 +35,7 @@ export function resolveFullTextTelemetryPolicy(value: unknown): FullTextTelemetr return result; } -export interface FullTextTelemetryHandle { - telemetry(): FullTextTelemetry; - configureTelemetry(options: ConfigureTelemetryOptions): void; -} +export type FullTextTelemetryHandle = NativeFullTextTelemetryHandle; export type FullTextTelemetrySample = { index: string; @@ -50,7 +48,7 @@ export type FullTextTelemetrySample = { }; type Scalar = string | number | boolean | null; -export type FullTextTelemetryRow = Record; +type FullTextTelemetryRow = Record; type FlatSnapshot = { counters: Map; gauges: FullTextTelemetryRow }; type Group = { index: string; @@ -267,6 +265,10 @@ export class FullTextTelemetryCollector { } } + get hasPendingRows(): boolean { + return this.#pending.size !== 0; + } + takeRows(): FullTextTelemetryRow[] { const rows = [...this.#pending.values()]; this.#pending.clear(); @@ -381,9 +383,14 @@ export class FullTextTelemetryCollector { const revision = this.#revision; if (this.#operator.enabled && stage !== 'closing') { let newest = entry; - for (const candidate of entry.group.entries) + let activeHandles = 0; + let closingHandles = 0; + for (const candidate of entry.group.entries) { + if (candidate.closing) closingHandles++; + else activeHandles++; if (!candidate.closing && candidate.flat && (newest.closing || candidate.observedAt > newest.observedAt)) newest = candidate; + } const row: FullTextTelemetryRow = { ...newest.flat?.gauges, metric: METRIC.FULL_TEXT, @@ -397,8 +404,8 @@ export class FullTextTelemetryCollector { collectionError: error ?? null, droppedRows: this.#droppedRows.toString(), observationAgeMilliseconds: newest.observedAt ? performance.now() - newest.observedAt : null, - activeHandles: [...entry.group.entries].filter((candidate) => !candidate.closing).length, - closingHandles: [...entry.group.entries].filter((candidate) => candidate.closing).length, + activeHandles, + closingHandles, available: !error && snapshot?.available === true, }; for (const [name, value] of entry.group.totals) row[name] = value.toString(); diff --git a/resources/analytics/fullTextTelemetryRuntime.ts b/resources/analytics/fullTextTelemetryRuntime.ts index 9328f1b340..9477e06f8d 100644 --- a/resources/analytics/fullTextTelemetryRuntime.ts +++ b/resources/analytics/fullTextTelemetryRuntime.ts @@ -53,7 +53,7 @@ export function acceptsFullTextTelemetryRow(row: Record): boole export function initializeFullTextTelemetry(): void { if (started) return; const listener = addAnalyticsListener((metrics) => { - if (!persistenceActive || !checkAnalyticsEnabled(false)) return; + if (!persistenceActive || !checkAnalyticsEnabled(false) || !fullTextTelemetryCollector.hasPendingRows) return; const rows = fullTextTelemetryCollector.takeRows(); metrics.push(...rows); }); From cafe1f1fa44fd4a3f49d6fc031b7d87cdeb01374 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Tue, 3 Nov 2026 07:12:58 -0700 Subject: [PATCH 06/26] test: report live telemetry activation state on timeout --- .../resources/nativeFullTextTelemetry.test.js | 14 +++++++++++++- .../nativeFullTextTelemetryWorkers.test.js | 13 ++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/unitTests/resources/nativeFullTextTelemetry.test.js b/unitTests/resources/nativeFullTextTelemetry.test.js index f4a997ab06..f965efc25c 100644 --- a/unitTests/resources/nativeFullTextTelemetry.test.js +++ b/unitTests/resources/nativeFullTextTelemetry.test.js @@ -458,7 +458,19 @@ describe('Harper full-text analytics lifecycle and persistence', function () { assert.strictEqual(live.scheduled, false); assert.deepStrictEqual(rows(), []); change(true); - await waitFor(() => live.collecting, 5000); + await waitFor(() => live.collecting, 5000).catch((error) => { + const watcher = require('#src/config/RootConfigWatcher').getSharedRootConfigWatcher(); + error.message += `; activation state ${JSON.stringify({ + file: YAML.parse(fs.readFileSync(file, 'utf8')).analytics?.fullText, + watched: watcher.config?.analytics?.fullText, + analyticsEnabled: analytics.checkAnalyticsEnabled(false), + armed: watcher._armedForTests, + polling: watcher._usingPollingForTests, + reads: watcher._readCountForTests, + listeners: watcher.listenerCount('change'), + })}`; + throw error; + }); await waitFor( () => rows().some((row) => row.method === 'writer') && rows().some((row) => row.method === 'reader'), 10000 diff --git a/unitTests/resources/nativeFullTextTelemetryWorkers.test.js b/unitTests/resources/nativeFullTextTelemetryWorkers.test.js index 2e072cb4f0..56b976dcfc 100644 --- a/unitTests/resources/nativeFullTextTelemetryWorkers.test.js +++ b/unitTests/resources/nativeFullTextTelemetryWorkers.test.js @@ -96,7 +96,18 @@ it('persists an actual managed-worker sample produced before main applies the sa assert.strictEqual(fullTextTelemetryCollector.collecting, false); await waitFor(() => Date.now() > produced.sampledAt); setRootConfigEntry('analytics', { ...readRootConfig().analytics, fullText: { enabled: true } }); - await waitFor(() => fullTextTelemetryCollector.collecting, 5000); + await waitFor(() => fullTextTelemetryCollector.collecting, 5000).catch((error) => { + error.message += `; activation state ${JSON.stringify({ + file: readRootConfig().analytics?.fullText, + watched: watcher.config?.analytics?.fullText, + analyticsEnabled: analytics.checkAnalyticsEnabled(false), + armed: watcher._armedForTests, + polling: watcher._usingPollingForTests, + reads: watcher._readCountForTests, + listeners: watcher.listenerCount('change'), + })}`; + throw error; + }); const mainEnabledAt = Date.now(); assert.ok(produced.sampledAt < mainEnabledAt); Atomics.store(barrier, 0, 1); From 9168792ab406d37e4878c5fcd6851f21df9b1c11 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Tue, 3 Nov 2026 08:50:03 -0700 Subject: [PATCH 07/26] fix: refine optional full-text telemetry initialization and sampling --- config/fullTextTelemetryConfiguration.ts | 7 +- resources/FULL_TEXT.md | 4 +- resources/analytics/DESIGN.md | 6 +- resources/analytics/fullTextTelemetry.ts | 64 +++++--- resources/indexes/fullTextNativeBinding.ts | 5 +- unitTests/build-tools/buildDist.test.mjs | 42 +++++ .../fullTextTelemetryConfiguration.test.js | 11 ++ .../resources/nativeFullTextTelemetry.test.js | 51 ++++++ .../nativeFullTextTelemetryClose.test.js | 148 ++++++++++++++++-- 9 files changed, 292 insertions(+), 46 deletions(-) diff --git a/config/fullTextTelemetryConfiguration.ts b/config/fullTextTelemetryConfiguration.ts index 6aae2e1c86..6ec9be3317 100644 --- a/config/fullTextTelemetryConfiguration.ts +++ b/config/fullTextTelemetryConfiguration.ts @@ -1,7 +1,7 @@ import { isDeepStrictEqual } from 'node:util'; import { getSharedRootConfigWatcher, type RootConfigWatcher } from './RootConfigWatcher.ts'; import { overlayRootEnvConfig } from './harperConfigEnvVars.ts'; -import { get, refreshConfigSubtree } from '../utility/environment/environmentManager.ts'; +import { refreshConfigSubtree } from '../utility/environment/environmentManager.ts'; import { resolveFullTextTelemetryPolicy } from '../resources/analytics/fullTextTelemetry.ts'; import { loggerWithTag } from '../utility/logging/logger.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; @@ -55,11 +55,6 @@ export function watchFullTextTelemetryConfiguration( watcher.off('available', changed); watcher.off('close', stop); } - try { - applyPolicy(get(PARAM)); - } catch { - report(); - } watcher.on('change', changed); watcher.on('unavailable', unavailable); watcher.on('available', changed); diff --git a/resources/FULL_TEXT.md b/resources/FULL_TEXT.md index 7c3e26f691..92ae994de0 100644 --- a/resources/FULL_TEXT.md +++ b/resources/FULL_TEXT.md @@ -207,7 +207,9 @@ analytics: The implied periodic interval is 60,000 milliseconds per handle. Calls are staggered across existing writer/readers, and their rows use Harper's existing analytics flush, storage, retention and protected `get_analytics` interface. Successful handle closes also collect final counters, so the interval is not a limit on total snapshot or export frequency. Collection never opens a cold index. To tune it, set `intervalMilliseconds` between 1,000 and 2,147,483,647; synchronous snapshots and export work have measurable cost, so qualify the intended number of indexes, reader churn and cadence. No production overhead bound is claimed. -Set `enabled: false` or remove the block to stop scheduled polling and persistent full-text export. A synchronous observation already executing completes. Editing unrelated settings does not restart collection. Invalid telemetry edits retain the last valid policy; deletion of the configuration file pauses file-based operator collection; environment overlays still apply while the file is missing. Disabling analytics also disables its full-text collection demand. Internal registered consumers can independently request observations; removing the last demand stops the timer. Consumer-only observations do not enable persistent analytics. +Set `enabled: false` or remove the block to stop scheduled polling and persistent full-text export. A synchronous observation already executing completes. Editing unrelated settings does not restart collection. Invalid telemetry edits retain the last valid policy. Deleting the configuration file removes its telemetry settings and retires in-process overrides whose matching file values disappeared; recreating the file does not restore those overrides. Environment overlays still apply while the file is missing. Disabling analytics also disables its full-text collection demand. Internal registered consumers can independently request observations; removing the last demand stops the timer. These are implementation hooks, not a public SDK, and currently have no production caller. Consumer-only observations do not enable persistent analytics. With operator export enabled, persisted fields reflect the union of operator and consumer recording requests. + +Workers and the receiving main thread enforce their current export policy independently. During configuration transitions, the receiver can discard worker rows, including a final close observation; these policy-filtered rows are not counted in `droppedRows` and may have no later replacement. `histograms: true` and `snapshotBytes: true` independently enable optional recording/reader-byte observation; both default to false. Turning collection off disables options no remaining consumer needs on live handles; a handle already closing keeps its last options until close settles. Re-enabling resumes cumulative counters. Enabling bytes schedules no native publication: values can remain pending until a successful publication/reload opportunity. Cached bytes, observation age and errors retain their own meanings; reader bytes are not directory totals or resident memory. diff --git a/resources/analytics/DESIGN.md b/resources/analytics/DESIGN.md index 60a735e4c0..56563e1c99 100644 --- a/resources/analytics/DESIGN.md +++ b/resources/analytics/DESIGN.md @@ -73,8 +73,8 @@ always a caller's measure. ## Full-text collection follows actual handle ownership -`fullTextTelemetry.ts` keeps private per-handle baselines and carries exact counters into a stable worker/index/role series. `nativeFullTextTelemetry.test.js` pins reader churn, wrap/precision, opt-in toggles and final close. The scheduler pauses before native close and observes final scalars only after settlement, so it never polls close in flight. Completed registrations release handle references; inactive groups remain only until their coalesced final row is drained. Pending export has a 128-row memory bound with explicit drop counts. +`fullTextTelemetry.ts` keeps private per-handle baselines and carries exact counters into a stable worker/index/role series. `nativeFullTextTelemetry.test.js` pins reader churn, wrap/precision, optional recording unions and peer-gauge aggregation; cached aggregate names include null leaves, so a live peer's queue age survives a newest null observation. `nativeFullTextTelemetryClose.test.js` holds backend close across live-peer timer ticks and recording changes: closing handles are neither sampled nor reconfigured until settlement. Completed registrations release references; inactive groups remain until their final row drains. Pending export has a 128-row bound with explicit drop counts. -`fullTextTelemetryRuntime.ts` joins removable internal consumer demand with live operator policy. Only operator demand enables persistent analytics. No demand means no telemetry timer; no cold handles are opened. Counters are decimal strings, rows are flat and `cumulative` makes the aggregation fold retain the newest sample, including availability/error fields, per worker. Lost final handle counts force a new series on the next successful live observation. Pending-row drops preserve cumulative totals while a group has live handles; dropping its final closed row can lose those totals. Overflow evicts by first pending insertion, and replacing a group's pending observation retains its position. Failed writer closes resume collection because Harper may keep the engine installed. Existing request metrics retain their averaging semantics. `aggregationCycle.test.js` verifies exact counters above 2^53 survive storage and `get_analytics`. +`fullTextTelemetryRuntime.ts` joins removable internal consumer demand with live operator policy. The consumer hook is an internal diagnostic/test seam with no production caller yet. Only operator demand enables persistence; persisted optional fields reflect the union of recording requests. No demand means no timer; no cold handles open. Exact counters are decimal strings; `cumulative` retains the newest worker row instead of averaging. Lost final counts force a new series on the next live observation. FIFO pending-row drops preserve totals only while a group remains live. Main also filters worker rows by its receive-time policy; transition losses, including final rows, do not increment `droppedRows`. Failed writer closes resume collection while the engine remains installed. `aggregationCycle.test.js` verifies counters above 2^53 survive storage and `get_analytics`. -`watchFullTextTelemetryConfiguration` uses the shared root watcher, environment overlay and transactional `refreshConfigSubtree`; unrelated edits preserve sampling cadence, invalid edits preserve valid policy, and availability events pause/recover after file deletion. `fullTextTelemetryConfiguration.test.js` names the effective override and recovery controls. Native runtime/query policy remains frozen separately. +`watchFullTextTelemetryConfiguration` initializes and reloads through the shared watcher's nested root, environment overlay and transactional `refreshConfigSubtree`; it never reads the missing camel-case flat aggregate. Invalid edits retain valid policy. Deletion retires overrides of matching removed file values; recreation does not restore them. Environment overlays continue to apply. `fullTextTelemetryConfiguration.test.js` pins initialization, override retirement and recovery. Native runtime/query policy remains frozen separately. Binding telemetry types are structural: `buildDist.test.mjs` checks compilation and emitted declarations without the optional package. diff --git a/resources/analytics/fullTextTelemetry.ts b/resources/analytics/fullTextTelemetry.ts index ed9e21c088..1955731bf3 100644 --- a/resources/analytics/fullTextTelemetry.ts +++ b/resources/analytics/fullTextTelemetry.ts @@ -1,8 +1,10 @@ import { createHash, randomUUID } from 'node:crypto'; -import type { ConfigureTelemetryOptions, FullTextTelemetry } from '@harperfast/fulltext/native'; import { METRIC } from './metadata.ts'; import type { NativeFullTextTelemetryHandle } from '../indexes/fullTextNativeBinding.ts'; +type ConfigureTelemetryOptions = Parameters[0]; +export type FullTextTelemetrySnapshot = ReturnType; + export type FullTextTelemetryPolicy = Required & { enabled: boolean; intervalMilliseconds: number; @@ -42,14 +44,18 @@ export type FullTextTelemetrySample = { role: 'writer' | 'reader'; series: string; sampledAt: number; - snapshot?: FullTextTelemetry; + snapshot?: FullTextTelemetrySnapshot; error?: 'configuration-error' | 'collection-error'; stage?: 'closing'; }; type Scalar = string | number | boolean | null; type FullTextTelemetryRow = Record; -type FlatSnapshot = { counters: Map; gauges: FullTextTelemetryRow }; +type FlatSnapshot = { + counters: Map; + gauges: FullTextTelemetryRow; + aggregateGauges: Map; +}; type Group = { index: string; role: 'writer' | 'reader'; @@ -86,14 +92,18 @@ function isCounter(path: string[]): boolean { return path[0] === 'native' && (path[1] === 'phases' || (path[1] === 'operations' && path[3] !== 'dispatched')); } -export function flattenFullTextTelemetry(snapshot: FullTextTelemetry): FlatSnapshot { +export function flattenFullTextTelemetry(snapshot: FullTextTelemetrySnapshot): FlatSnapshot { const counters = new Map(); const gauges: FullTextTelemetryRow = {}; + const aggregateGauges = new Map(); function visit(value: unknown, path: string[]) { const name = path.join('_'); if (typeof value === 'bigint') { - if (isCounter(path)) counters.set(name, (counters.get(name) ?? 0n) + value); - else gauges[name] = value.toString(); + if (isCounter(path)) { + counters.set(name, (counters.get(name) ?? 0n) + value); + return; + } + gauges[name] = value.toString(); } else if (value === null || typeof value === 'boolean' || typeof value === 'number') { gauges[name] = value as Scalar; } else if (typeof value === 'string') { @@ -103,15 +113,22 @@ export function flattenFullTextTelemetry(snapshot: FullTextTelemetry): FlatSnaps else gauges[name] = path.includes('unsupported') ? 'unsupported' : 'unavailable'; } else if (Array.isArray(value)) { for (let i = 0; i < value.length; i++) visit(value[i], [...path, String(i)]); + return; } else if (value && typeof value === 'object') { for (const [key, child] of Object.entries(value)) { const boundedKey = path.at(-1) === 'errors' && !/^E_[A-Z_]{1,60}$/.test(key) ? 'other' : key; visit(child, [...path, boundedKey]); } - } + return; + } else return; + if ( + /^native_operations_[A-Za-z]+_dispatched$/.test(name) || + /^native_queues_[A-Za-z]+_(commands|bytes|maxCommands|maxBytes|oldestAgeNanoseconds)$/.test(name) + ) + aggregateGauges.set(name, typeof value === 'bigint' && value >= 0n ? value : null); } visit(snapshot, []); - return { counters, gauges }; + return { counters, gauges, aggregateGauges }; } export function fullTextTelemetryIndexLabel(database: string, table: string, index: string): string { @@ -378,7 +395,12 @@ export class FullTextTelemetryCollector { } } - #publish(entry: Entry, snapshot?: FullTextTelemetry, error?: FullTextTelemetrySample['error'], stage?: 'closing') { + #publish( + entry: Entry, + snapshot?: FullTextTelemetrySnapshot, + error?: FullTextTelemetrySample['error'], + stage?: 'closing' + ) { const sampledAt = Date.now(); const revision = this.#revision; if (this.#operator.enabled && stage !== 'closing') { @@ -409,23 +431,17 @@ export class FullTextTelemetryCollector { available: !error && snapshot?.available === true, }; for (const [name, value] of entry.group.totals) row[name] = value.toString(); - for (const name of Object.keys(row)) { - if ( - /^native_operations_[A-Za-z]+_dispatched$/.test(name) || - /^native_queues_[A-Za-z]+_(commands|bytes|maxCommands|maxBytes|oldestAgeNanoseconds)$/.test(name) - ) { - let total = 0n; - let observed = false; - for (const candidate of entry.group.entries) { - const value = candidate.flat?.gauges[name]; - if (!candidate.closing && typeof value === 'string' && /^\d+$/.test(value)) { - const number = BigInt(value); - total = name.endsWith('oldestAgeNanoseconds') ? (number > total ? number : total) : total + number; - observed = true; - } + for (const name of newest.flat?.aggregateGauges.keys() ?? []) { + let total = 0n; + let observed = false; + for (const candidate of entry.group.entries) { + const value = candidate.flat?.aggregateGauges.get(name); + if (!candidate.closing && typeof value === 'bigint') { + total = name.endsWith('oldestAgeNanoseconds') ? (value > total ? value : total) : total + value; + observed = true; } - row[name] = observed ? total.toString() : null; } + row[name] = observed ? total.toString() : null; } if (!this.#pending.has(entry.group) && this.#pending.size >= this.#maxPendingRows) { const oldest = this.#pending.keys().next().value!; diff --git a/resources/indexes/fullTextNativeBinding.ts b/resources/indexes/fullTextNativeBinding.ts index bd086bba12..842585747d 100644 --- a/resources/indexes/fullTextNativeBinding.ts +++ b/resources/indexes/fullTextNativeBinding.ts @@ -1,4 +1,3 @@ -import type { FullTextTelemetry, ConfigureTelemetryOptions } from '@harperfast/fulltext/native'; import type { FullTextDerivedIndexEngine } from './fullTextDerivedIndex.ts'; import * as env from '../../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS } from '../../utility/hdbTerms.ts'; @@ -98,8 +97,8 @@ export type NativeFullTextTraceResult = { }; export interface NativeFullTextTelemetryHandle { - telemetry(): FullTextTelemetry; - configureTelemetry(options: ConfigureTelemetryOptions): void; + telemetry(): { available: boolean }; + configureTelemetry(options: { histograms?: boolean; snapshotBytes?: boolean }): void; } export interface NativeFullTextReader extends NativeFullTextTelemetryHandle { diff --git a/unitTests/build-tools/buildDist.test.mjs b/unitTests/build-tools/buildDist.test.mjs index 619cadaca8..caaf97b0be 100644 --- a/unitTests/build-tools/buildDist.test.mjs +++ b/unitTests/build-tools/buildDist.test.mjs @@ -3,6 +3,7 @@ import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { SourceMap } from 'node:module'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; import ts from 'typescript'; import { build, escapeNonLatin1Literals, findNonLatin1 } from '../../build-tools/build-dist.mjs'; @@ -110,3 +111,44 @@ describe('build-dist non-Latin-1 literal escaping', function () { } }); }); + +it('builds native telemetry declarations without the optional fulltext package', function () { + this.timeout(30_000); + const root = fileURLToPath(new URL('../../', import.meta.url)); + const configPath = join(root, 'tsconfig.build.json'); + const configFile = ts.readConfigFile(configPath, ts.sys.readFile); + const config = ts.parseJsonConfigFileContent(configFile.config, ts.sys, root); + const options = { ...config.options, noEmit: false, declaration: true, emitDeclarationOnly: true }; + const host = ts.createCompilerHost(options); + const optionalPackage = /[/\\]node_modules[/\\]@harperfast[/\\]fulltext(?:[/\\]|$)/; + for (const method of ['fileExists', 'directoryExists', 'readFile']) { + const original = host[method].bind(host); + host[method] = (file) => + optionalPackage.test(file) ? (method === 'readFile' ? undefined : false) : original(file); + } + const files = ['resources/indexes/fullTextNativeBinding.ts', 'resources/analytics/fullTextTelemetry.ts'].map((file) => + join(root, file) + ); + const program = ts.createProgram(files, options, host); + for (const file of files) { + const source = program.getSourceFile(file); + assert.deepStrictEqual( + program + .getSemanticDiagnostics(source) + .map((diagnostic) => ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n')), + [] + ); + const declarations = []; + const emitted = program.emit( + source, + (name, text) => { + if (name.endsWith('.d.ts')) declarations.push(text); + }, + undefined, + true + ); + assert.strictEqual(emitted.emitSkipped, false); + assert.strictEqual(declarations.length, 1); + assert.doesNotMatch(declarations[0], /@harperfast\/fulltext/); + } +}); diff --git a/unitTests/config/fullTextTelemetryConfiguration.test.js b/unitTests/config/fullTextTelemetryConfiguration.test.js index f3ce1d6e45..79125d41c0 100644 --- a/unitTests/config/fullTextTelemetryConfiguration.test.js +++ b/unitTests/config/fullTextTelemetryConfiguration.test.js @@ -51,6 +51,17 @@ describe('Live full-text telemetry configuration', function () { watcher = new RootConfigWatcher(file); stop = watchFullTextTelemetryConfiguration((policy) => samples.push(policy), watcher); } + it('initializes from the ready nested file policy without an eager aggregate/default application', async () => { + writeFileSync(file, stringify(root({ enabled: true, histograms: true }))); + watcher = new RootConfigWatcher(file); + await watcher.ready; + const samples = []; + stop = watchFullTextTelemetryConfiguration((policy) => samples.push(policy), watcher); + assert.deepStrictEqual(samples, []); + await waitFor(() => samples.length === 1); + assert.strictEqual(samples[0].enabled, true); + assert.strictEqual(samples[0].histograms, true); + }); it('rejects quoted telemetry values consistently at validation and live application', () => { for (const value of [ { enabled: 'true' }, diff --git a/unitTests/resources/nativeFullTextTelemetry.test.js b/unitTests/resources/nativeFullTextTelemetry.test.js index f965efc25c..52357d77ec 100644 --- a/unitTests/resources/nativeFullTextTelemetry.test.js +++ b/unitTests/resources/nativeFullTextTelemetry.test.js @@ -163,6 +163,13 @@ describe('Published full-text telemetry collection', function () { collector.collect(index); assert.strictEqual(samples.length, 1); assert.deepStrictEqual(collector.takeRows(), []); + collector.setOperatorPolicy({ enabled: true }); + collector.collect(index); + const combined = row('writer'); + assert.strictEqual(combined.native_histogramsEnabled, true); + assert.strictEqual(combined.native_snapshotBytesEnabled, true); + collector.setOperatorPolicy({ enabled: false }); + assert.strictEqual(collector.collecting, true); removeHistogram(); assert.strictEqual(index.telemetry().native.histogramsEnabled, false); assert.strictEqual(index.telemetry().native.snapshotBytesEnabled, true); @@ -350,6 +357,50 @@ describe('Published full-text telemetry collection', function () { assert.strictEqual(unavailable.gauges.native_reader_documents, undefined); assert.strictEqual(unavailable.counters.get('logicalBatches_completed'), 1n); }); + it('preserves peer gauge sums, null ages, newest-key selection and closing exclusions', async () => { + const { index } = await writer(); + const snapshots = [index.telemetry(), index.telemetry()]; + for (let i = 0; i < snapshots.length; i++) { + const native = snapshots[i].native; + native.operations.search.dispatched = BigInt(i + 2); + native.queues.ordinarySearch.commands = BigInt(i + 3); + native.queues.ordinarySearch.bytes = BigInt(i + 4); + native.queues.ordinarySearch.oldestAgeNanoseconds = i === 0 ? 7_000_000n : null; + native.reader.documents = BigInt(i + 1); + } + const peers = snapshots.map((snapshot) => ({ telemetry: () => snapshot, configureTelemetry() {} })); + collector.setOperatorPolicy({ enabled: true }); + for (const peer of peers) { + collector.register(peer, 'catalog.Product/search', 'reader'); + collector.collect(peer); + } + const combined = row(); + assert.strictEqual(combined.native_operations_search_dispatched, '5'); + assert.strictEqual(combined.native_queues_ordinarySearch_commands, '7'); + assert.strictEqual(combined.native_queues_ordinarySearch_bytes, '9'); + assert.strictEqual(combined.native_queues_ordinarySearch_oldestAgeNanoseconds, '7000000'); + assert.strictEqual(combined.native_reader_documents, '2'); + snapshots[1].native.queues.ordinarySearch.oldestAgeNanoseconds = 9_000_000n; + collector.collect(peers[1]); + assert.strictEqual(row().native_queues_ordinarySearch_oldestAgeNanoseconds, '9000000'); + delete snapshots[1].native.queues.ordinarySearch.bytes; + collector.collect(peers[1]); + assert.strictEqual(Object.hasOwn(row(), 'native_queues_ordinarySearch_bytes'), false); + snapshots[1].native.queues.ordinarySearch.bytes = null; + collector.collect(peers[1]); + assert.strictEqual(row().native_queues_ordinarySearch_bytes, '4'); + snapshots[0].native.queues.ordinarySearch.bytes = -1n; + collector.collect(peers[0]); + collector.collect(peers[1]); + assert.strictEqual(row().native_queues_ordinarySearch_bytes, null); + collector.beforeClose(peers[0]); + snapshots[1].native.queues.ordinarySearch.oldestAgeNanoseconds = null; + collector.collect(peers[1]); + const retiring = row(); + assert.strictEqual(retiring.native_operations_search_dispatched, '3'); + assert.strictEqual(retiring.native_queues_ordinarySearch_commands, '4'); + assert.strictEqual(retiring.native_queues_ordinarySearch_oldestAgeNanoseconds, null); + }); it('validates optional configuration without adding an enabled default', () => { assert.strictEqual(resolveFullTextTelemetryPolicy(undefined).enabled, false); for (const value of [ diff --git a/unitTests/resources/nativeFullTextTelemetryClose.test.js b/unitTests/resources/nativeFullTextTelemetryClose.test.js index 67008b6ae0..7cb889ed25 100644 --- a/unitTests/resources/nativeFullTextTelemetryClose.test.js +++ b/unitTests/resources/nativeFullTextTelemetryClose.test.js @@ -11,7 +11,20 @@ const { encodeFullTextCursorPayload, } = require('#src/resources/indexes/fullTextDerivedIndex'); const { DERIVED_INDEX_ACCEPTED } = require('#src/resources/derivedIndexRuntime'); -const { subscribeFullTextTelemetry } = require('#src/resources/analytics/fullTextTelemetryRuntime'); +const { + subscribeFullTextTelemetry, + registerNativeFullTextTelemetry, + fullTextTelemetryCollector, +} = require('#src/resources/analytics/fullTextTelemetryRuntime'); + +const limits = { + indexingThreads: 2, + searchThreads: 2, + writerMemoryBytes: 64 * 1024 * 1024, + maxQueuedCommands: 16, + maxQueuedBytes: 64 * 1024 * 1024, + maxBatchBytes: 8 * 1024 * 1024, +}; it('keeps observing an installed real writer after a rejected require-clean close and closes on retry', async function () { this.timeout(20_000); @@ -26,14 +39,7 @@ it('keeps observing an installed real writer after a rejected require-clean clos fields: [{ name: 'title' }], analyzer: 'english@2', positions: true, - limits: { - indexingThreads: 2, - searchThreads: 2, - writerMemoryBytes: 64 * 1024 * 1024, - maxQueuedCommands: 16, - maxQueuedBytes: 64 * 1024 * 1024, - maxBatchBytes: 8 * 1024 * 1024, - }, + limits, binding, }); await lifecycle.initialize({ reclaimRetired: false }); @@ -114,3 +120,127 @@ it('keeps observing an installed real writer after a rejected require-clean clos } } }); + +it('does not poll or reconfigure a real writer while backend close spans scheduled peer observations', async function () { + this.timeout(40_000); + if (!(await supportsPublishedFullTextBinding())) this.skip(); + const binding = await loadFullTextNativeBinding(); + const id = `close-poll-${process.pid}`; + const lifecycle = new NativeFullTextDerivedIndexLifecycle({ + storePath: join(setupTestDBPath(), id), + storeName: 'catalog.Product.title', + indexId: id, + sourceGeneration: 'generation-1', + fields: [{ name: 'title' }], + analyzer: 'english@2', + positions: true, + limits, + binding, + }); + await lifecycle.initialize({ reclaimRetired: false }); + let writer, witness, latest, closingAt, witnessAfterClose, shutdown, removeHistograms; + let writerSamples = 0; + let releaseClose; + const gate = new Promise((resolve) => (releaseClose = resolve)); + const remove = subscribeFullTextTelemetry( + (sample) => { + if (sample.index === id) { + if (sample.stage === 'closing') closingAt = performance.now(); + if (sample.snapshot?.available) { + latest = sample.snapshot; + writerSamples++; + } + } else if ( + sample.index === id + '-peer' && + closingAt !== undefined && + performance.now() - closingAt >= 1000 && + sample.snapshot?.native.histogramsEnabled + ) + witnessAfterClose = sample; + }, + { intervalMilliseconds: 1000 } + ); + const backend = new FullTextDerivedIndexBackend({ + id, + lifecycle: { + inspect: () => lifecycle.inspect(), + reset: (epoch) => lifecycle.reset(epoch), + async open() { + writer = await lifecycle.open(); + return writer; + }, + }, + closeTimeoutMilliseconds: 15_000, + shutdownTimeoutMilliseconds: 30_000, + }); + backend.attach({ + isOwnerEpoch: (epoch) => epoch === 1n, + getReadiness: () => ({ state: 'ready', ownerEpoch: 1n, rebuildAttempts: 0 }), + publicationChanged() {}, + }); + try { + backend.getDurableCursor(); + assert.strictEqual( + backend.deliver({ + ownerEpoch: 1n, + transactions: [], + records: [], + through: { format: 1, logs: { local: 1 } }, + bytes: 0, + }), + DERIVED_INDEX_ACCEPTED + ); + backend.flush(); + await waitFor(() => backend.getDurableCursor()?.logs.local === 1, 5000); + witness = await binding.openNativeFullTextIndex({ + path: join(setupTestDBPath(), id + '-peer'), + indexId: id + '-peer', + generation: 'g1', + fields: [{ name: 'title' }], + analyzer: 'english@2', + positions: true, + limits, + }); + registerNativeFullTextTelemetry(witness, id + '-peer', 'writer'); + const close = writer.close.bind(writer); + writer.close = async (options) => { + await gate; + return close(options); + }; + shutdown = backend.shutdown(1n); + void shutdown.catch(() => {}); + await waitFor(() => closingAt !== undefined); + const before = writerSamples; + removeHistograms = subscribeFullTextTelemetry(() => {}, { histograms: true, intervalMilliseconds: 1000 }); + await waitFor(() => witnessAfterClose, 5000); + assert.strictEqual(writerSamples, before, 'the scheduler must leave the closing writer unsampled'); + assert.strictEqual(writer.telemetry().native.histogramsEnabled, false); + releaseClose(); + await shutdown; + assert.strictEqual(writerSamples, before + 1, 'successful close must emit exactly one final observation'); + assert.strictEqual(latest.native.state, 'closed'); + assert.strictEqual(latest.native.histogramsEnabled, false); + } finally { + releaseClose(); + try { + await backend.shutdown(1n); + if (writer && writer.status().state !== 'closed') await writer.close({ mode: 'rollback' }); + } finally { + try { + if (witness) { + fullTextTelemetryCollector.beforeClose(witness); + try { + await witness.close({ mode: 'rollback' }); + fullTextTelemetryCollector.afterClose(witness); + } catch (error) { + fullTextTelemetryCollector.closeFailed(witness); + throw error; + } + } + } finally { + removeHistograms?.(); + remove(); + } + } + } +}); From e148d8f09a6268354a812b4b5289a884510419fc Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Tue, 3 Nov 2026 09:16:33 -0700 Subject: [PATCH 08/26] test: cover all optional telemetry imports and narrow gauge classification --- config/DESIGN.md | 2 +- resources/FULL_TEXT.md | 2 +- resources/analytics/DESIGN.md | 2 +- resources/analytics/fullTextTelemetry.ts | 7 +++++-- unitTests/build-tools/buildDist.test.mjs | 9 +++++++++ 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/config/DESIGN.md b/config/DESIGN.md index bf2480a239..a87e59df7f 100644 --- a/config/DESIGN.md +++ b/config/DESIGN.md @@ -240,7 +240,7 @@ off on the very boot that could not read its configuration. ## `RootConfigWatcher` schedules a trailing verify-read (`config/RootConfigWatcher.ts`) -Chokidar throttles `EV.CHANGE` events per path for 50ms (`node_modules/chokidar/index.js`) with no trailing emission, so the second of two `atomicWriteFile()` renames landing within 50ms produces no watcher event. After a successful `handleChange` read, `RootConfigWatcher` arms one 100ms (`TRAILING_VERIFY_READ_DELAY_MS`, double the 50ms throttle window) trailing verify-read (`#read(true)`), re-armed on each event and cleared on `close()`, terminal read failure, or when `ConfigReadRetry` takes over. Post-`ready` reads short-circuit when raw file text matches `#lastData` and otherwise compare the parsed object against an unmutated clone (`#lastParsedConfig`) with `isDeepStrictEqual`, so neither the trailing read nor an identical rewrite emits a duplicate `change` even when consumers (`harper_logger`) mutate `this.#config` in place (`unitTests/config/rootConfigWatcher.test.js`). Root-config `OptionsWatcher`s lack this trailing read. `unlink` emits contained `unavailable`; the first successful recovery read emits `available` even when file text is unchanged. Telemetry uses these events to pause file-based policy and reapply environment overlays; invalid recreation retains the last valid policy (`unitTests/config/fullTextTelemetryConfiguration.test.js`). +Chokidar throttles `EV.CHANGE` events per path for 50ms (`node_modules/chokidar/index.js`) with no trailing emission, so the second of two `atomicWriteFile()` renames landing within 50ms produces no watcher event. After a successful `handleChange` read, `RootConfigWatcher` arms one 100ms (`TRAILING_VERIFY_READ_DELAY_MS`, double the 50ms throttle window) trailing verify-read (`#read(true)`), re-armed on each event and cleared on `close()`, terminal read failure, or when `ConfigReadRetry` takes over. Post-`ready` reads short-circuit when raw file text matches `#lastData` and otherwise compare the parsed object against an unmutated clone (`#lastParsedConfig`) with `isDeepStrictEqual`, so neither the trailing read nor an identical rewrite emits a duplicate `change` even when consumers (`harper_logger`) mutate `this.#config` in place (`unitTests/config/rootConfigWatcher.test.js`). Root-config `OptionsWatcher`s lack this trailing read. `unlink` emits contained `unavailable`; the first successful recovery read emits `available` even when file text is unchanged. Telemetry removes file policy and reapplies environment overlays. Overrides retire where matching effective values change; recreation does not restore them. Invalid recreation retains the last valid policy (`unitTests/config/fullTextTelemetryConfiguration.test.js`). ## `RootConfigWatcher` polls under Bun (`config/RootConfigWatcher.ts`) diff --git a/resources/FULL_TEXT.md b/resources/FULL_TEXT.md index 92ae994de0..da2e67ea04 100644 --- a/resources/FULL_TEXT.md +++ b/resources/FULL_TEXT.md @@ -207,7 +207,7 @@ analytics: The implied periodic interval is 60,000 milliseconds per handle. Calls are staggered across existing writer/readers, and their rows use Harper's existing analytics flush, storage, retention and protected `get_analytics` interface. Successful handle closes also collect final counters, so the interval is not a limit on total snapshot or export frequency. Collection never opens a cold index. To tune it, set `intervalMilliseconds` between 1,000 and 2,147,483,647; synchronous snapshots and export work have measurable cost, so qualify the intended number of indexes, reader churn and cadence. No production overhead bound is claimed. -Set `enabled: false` or remove the block to stop scheduled polling and persistent full-text export. A synchronous observation already executing completes. Editing unrelated settings does not restart collection. Invalid telemetry edits retain the last valid policy. Deleting the configuration file removes its telemetry settings and retires in-process overrides whose matching file values disappeared; recreating the file does not restore those overrides. Environment overlays still apply while the file is missing. Disabling analytics also disables its full-text collection demand. Internal registered consumers can independently request observations; removing the last demand stops the timer. These are implementation hooks, not a public SDK, and currently have no production caller. Consumer-only observations do not enable persistent analytics. With operator export enabled, persisted fields reflect the union of operator and consumer recording requests. +Set `enabled: false` or remove the block to stop scheduled polling and persistent full-text export. A synchronous observation already executing completes. Editing unrelated settings does not restart collection. Invalid telemetry edits retain the last valid policy. Deleting the configuration file removes its telemetry settings. An in-process override is retired if deletion changes its matching configuration value after environment overlays are applied; recreating the file does not restore retired overrides. Environment overlays still apply while the file is missing. Disabling analytics also disables its full-text collection demand. Internal registered consumers can independently request observations; removing the last demand stops the timer. These are implementation hooks, not a public SDK, and currently have no production caller. Consumer-only observations do not enable persistent analytics. With operator export enabled, persisted fields reflect the union of operator and consumer recording requests. Workers and the receiving main thread enforce their current export policy independently. During configuration transitions, the receiver can discard worker rows, including a final close observation; these policy-filtered rows are not counted in `droppedRows` and may have no later replacement. diff --git a/resources/analytics/DESIGN.md b/resources/analytics/DESIGN.md index 56563e1c99..8a064f3fd3 100644 --- a/resources/analytics/DESIGN.md +++ b/resources/analytics/DESIGN.md @@ -77,4 +77,4 @@ always a caller's measure. `fullTextTelemetryRuntime.ts` joins removable internal consumer demand with live operator policy. The consumer hook is an internal diagnostic/test seam with no production caller yet. Only operator demand enables persistence; persisted optional fields reflect the union of recording requests. No demand means no timer; no cold handles open. Exact counters are decimal strings; `cumulative` retains the newest worker row instead of averaging. Lost final counts force a new series on the next live observation. FIFO pending-row drops preserve totals only while a group remains live. Main also filters worker rows by its receive-time policy; transition losses, including final rows, do not increment `droppedRows`. Failed writer closes resume collection while the engine remains installed. `aggregationCycle.test.js` verifies counters above 2^53 survive storage and `get_analytics`. -`watchFullTextTelemetryConfiguration` initializes and reloads through the shared watcher's nested root, environment overlay and transactional `refreshConfigSubtree`; it never reads the missing camel-case flat aggregate. Invalid edits retain valid policy. Deletion retires overrides of matching removed file values; recreation does not restore them. Environment overlays continue to apply. `fullTextTelemetryConfiguration.test.js` pins initialization, override retirement and recovery. Native runtime/query policy remains frozen separately. Binding telemetry types are structural: `buildDist.test.mjs` checks compilation and emitted declarations without the optional package. +`watchFullTextTelemetryConfiguration` initializes and reloads through the shared watcher's nested root, environment overlay and transactional `refreshConfigSubtree`; it never reads the missing camel-case flat aggregate. Invalid edits retain valid policy. Deletion retires overrides only where matching values change after environment overlays; recreation does not restore them. `fullTextTelemetryConfiguration.test.js` pins initialization, override retirement and recovery. Native runtime/query policy remains frozen separately. Binding telemetry types are structural: `buildDist.test.mjs` checks compilation and emitted declarations without the optional package. diff --git a/resources/analytics/fullTextTelemetry.ts b/resources/analytics/fullTextTelemetry.ts index 1955731bf3..76473f2f6f 100644 --- a/resources/analytics/fullTextTelemetry.ts +++ b/resources/analytics/fullTextTelemetry.ts @@ -122,8 +122,11 @@ export function flattenFullTextTelemetry(snapshot: FullTextTelemetrySnapshot): F return; } else return; if ( - /^native_operations_[A-Za-z]+_dispatched$/.test(name) || - /^native_queues_[A-Za-z]+_(commands|bytes|maxCommands|maxBytes|oldestAgeNanoseconds)$/.test(name) + path.length === 4 && + path[0] === 'native' && + (path[1] === 'operations' || path[1] === 'queues') && + (/^native_operations_[A-Za-z]+_dispatched$/.test(name) || + /^native_queues_[A-Za-z]+_(commands|bytes|maxCommands|maxBytes|oldestAgeNanoseconds)$/.test(name)) ) aggregateGauges.set(name, typeof value === 'bigint' && value >= 0n ? value : null); } diff --git a/unitTests/build-tools/buildDist.test.mjs b/unitTests/build-tools/buildDist.test.mjs index caaf97b0be..a993d0edd4 100644 --- a/unitTests/build-tools/buildDist.test.mjs +++ b/unitTests/build-tools/buildDist.test.mjs @@ -118,6 +118,15 @@ it('builds native telemetry declarations without the optional fulltext package', const configPath = join(root, 'tsconfig.build.json'); const configFile = ts.readConfigFile(configPath, ts.sys.readFile); const config = ts.parseJsonConfigFileContent(configFile.config, ts.sys, root); + const optionalImports = config.fileNames.flatMap((file) => + ts + .preProcessFile(ts.sys.readFile(file), true, true) + .importedFiles.filter( + ({ fileName }) => fileName === '@harperfast/fulltext' || fileName.startsWith('@harperfast/fulltext/') + ) + .map(({ fileName }) => `${file}: ${fileName}`) + ); + assert.deepStrictEqual(optionalImports, []); const options = { ...config.options, noEmit: false, declaration: true, emitDeclarationOnly: true }; const host = ts.createCompilerHost(options); const optionalPackage = /[/\\]node_modules[/\\]@harperfast[/\\]fulltext(?:[/\\]|$)/; From efcf56f31dee3dec12f567e480837966b8e9e677 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Tue, 3 Nov 2026 09:20:48 -0700 Subject: [PATCH 09/26] docs: keep configuration design notes within the CI budget --- config/DESIGN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/DESIGN.md b/config/DESIGN.md index a87e59df7f..8e3e243f34 100644 --- a/config/DESIGN.md +++ b/config/DESIGN.md @@ -240,7 +240,7 @@ off on the very boot that could not read its configuration. ## `RootConfigWatcher` schedules a trailing verify-read (`config/RootConfigWatcher.ts`) -Chokidar throttles `EV.CHANGE` events per path for 50ms (`node_modules/chokidar/index.js`) with no trailing emission, so the second of two `atomicWriteFile()` renames landing within 50ms produces no watcher event. After a successful `handleChange` read, `RootConfigWatcher` arms one 100ms (`TRAILING_VERIFY_READ_DELAY_MS`, double the 50ms throttle window) trailing verify-read (`#read(true)`), re-armed on each event and cleared on `close()`, terminal read failure, or when `ConfigReadRetry` takes over. Post-`ready` reads short-circuit when raw file text matches `#lastData` and otherwise compare the parsed object against an unmutated clone (`#lastParsedConfig`) with `isDeepStrictEqual`, so neither the trailing read nor an identical rewrite emits a duplicate `change` even when consumers (`harper_logger`) mutate `this.#config` in place (`unitTests/config/rootConfigWatcher.test.js`). Root-config `OptionsWatcher`s lack this trailing read. `unlink` emits contained `unavailable`; the first successful recovery read emits `available` even when file text is unchanged. Telemetry removes file policy and reapplies environment overlays. Overrides retire where matching effective values change; recreation does not restore them. Invalid recreation retains the last valid policy (`unitTests/config/fullTextTelemetryConfiguration.test.js`). +Chokidar throttles change events for each path for 50ms without a trailing emit, so closely spaced atomic renames can go unseen. After a successful read, `RootConfigWatcher` rearms a 100ms verify-read (`#read(true)`); close, terminal failure or `ConfigReadRetry` clears it. Unchanged raw text skips parsing; parsed changes use the immutable `#lastParsedConfig` so consumer mutations cannot trigger duplicate events (`rootConfigWatcher.test.js`). Root-config `OptionsWatcher`s lack this verification. Unlink emits contained `unavailable`; the first recovery emits `available` even for unchanged text. Telemetry reapplies environment overlays without file policy and retires overrides where effective values change; recreation cannot restore retired overrides. Invalid recreation retains valid policy (`fullTextTelemetryConfiguration.test.js`). ## `RootConfigWatcher` polls under Bun (`config/RootConfigWatcher.ts`) From cd5ced8d63930bb178fb201f375da425385fea73 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Tue, 3 Nov 2026 10:14:16 -0700 Subject: [PATCH 10/26] docs: document full-text benchmark usage and ignore generated results --- .gitignore | 3 +- benchmarks/README.md | 70 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 3eae7855b2..0feb20ffd2 100644 --- a/.gitignore +++ b/.gitignore @@ -52,10 +52,11 @@ harper-*.tgz .claude/ cache/ -# YCSB benchmark results (generated) +# Benchmark results (generated) benchmarks/ycsb/results/ benchmarks/ycsb/bench-out/ benchmarks/compression/results/ +benchmarks/fulltext-analytics/results/ # export_local integration-test artifacts — these tests pass `path: './'`, so the export lands in # the repo root and a `git add -A` after a test run commits it diff --git a/benchmarks/README.md b/benchmarks/README.md index e599ab42dc..583b5e0128 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -2,18 +2,19 @@ This directory contains single-node storage and throughput benchmarks for Harper. -| Benchmark | File | What it measures | -| ------------------ | --------------------------------- | ----------------------------------------------------------------------------------------------------------- | -| YCSB | `ycsb/` | Standard CRUD workloads (A–F) across the REST interface | -| HNSW search | `hnsw-search.js` | In-memory vector index search latency and recall | -| **Indexed-write** | `indexed-write/` | Write throughput at 0 / 3 / 5 secondary indexes (**ST-2**) | -| **TTL-churn** | `ttl-churn/` | Storage size stability under continuous insert-with-TTL (**ST-1**) | -| **Concurrent R+W** | `concurrent-rw/` | Read p99 under mixed concurrent writes on a highly-indexed table (**ST-5**) | -| Hot-key commit | `hot-key-commit/` | Commit throughput, latency and retries per commit when audited transactions contend on a few keys | -| **SQL engine A/B** | `sql-engine/` | New (Resource-API) vs legacy (AlaSQL) SQL engine latency, per query shape | -| **Compression** | `compression/` | RocksDB codec comparison: on-disk size and throughput per codec | -| WebSocket scale | [`ws-scale/`](ws-scale/README.md) | Memory and CPU per WebSocket connection, subscription and delivered message; write rates that drive fan-out | -| Lockdown | `lockdown/` | Per-mode `applications.lockdown` cost in a worker: holey-array string building, msgpackr decode | +| Benchmark | File | What it measures | +| ------------------- | --------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| YCSB | `ycsb/` | Standard CRUD workloads (A–F) across the REST interface | +| HNSW search | `hnsw-search.js` | In-memory vector index search latency and recall | +| Full-text analytics | `fulltext-analytics.mjs` | Paired collection-disabled/enabled cost through real Tables, native indexes and analytics persistence | +| **Indexed-write** | `indexed-write/` | Write throughput at 0 / 3 / 5 secondary indexes (**ST-2**) | +| **TTL-churn** | `ttl-churn/` | Storage size stability under continuous insert-with-TTL (**ST-1**) | +| **Concurrent R+W** | `concurrent-rw/` | Read p99 under mixed concurrent writes on a highly-indexed table (**ST-5**) | +| Hot-key commit | `hot-key-commit/` | Commit throughput, latency and retries per commit when audited transactions contend on a few keys | +| **SQL engine A/B** | `sql-engine/` | New (Resource-API) vs legacy (AlaSQL) SQL engine latency, per query shape | +| **Compression** | `compression/` | RocksDB codec comparison: on-disk size and throughput per codec | +| WebSocket scale | [`ws-scale/`](ws-scale/README.md) | Memory and CPU per WebSocket connection, subscription and delivered message; write rates that drive fan-out | +| Lockdown | `lockdown/` | Per-mode `applications.lockdown` cost in a worker: holey-array string building, msgpackr decode | The three new benchmarks (ST-1, ST-2, ST-5) address gaps called out in §6.3 of the Harper Release Testing Strategy and §5 of the v5 Integration Test Plan. @@ -43,6 +44,51 @@ npx harper-integration-test-setup-loopback --- +## Full-text analytics (`fulltext-analytics.mjs`) + +Run manually after `npm run build`, with the optional `@harperfast/fulltext` native +package installed on a supported platform. The runner starts isolated Mocha children +using Harper's test fixtures and RocksDB. It does not start a REST server or require +loopback setup. Each pair alternates which profile runs first. + +### Short smoke (not performance qualification) + +```sh +node benchmarks/fulltext-analytics.mjs --duration 1000 --indexes 1 --pairs 1 --interval 1000 --output benchmarks/fulltext-analytics/results/smoke-01 +``` + +### Full-cadence comparison (about five minutes) + +```sh +node benchmarks/fulltext-analytics.mjs --output benchmarks/fulltext-analytics/results/paired-01 +``` + +| Flag | Default | Meaning | +| --------------- | -------- | ----------------------------------------------------------------------------------------- | +| `--duration` | `70000` | Measured milliseconds per profile; 100–3600000 | +| `--indexes` | `4` | Audited product Tables with native full-text indexes; 1–16 | +| `--pairs` | `2` | Disabled/enabled pairs; 1–10 | +| `--interval` | `60000` | Telemetry collection interval in milliseconds; 1000–2147483647 | +| `--maxIndexLag` | `0` | Allowed index lag in milliseconds; 0–60000. Zero waits up to 5000 ms for index freshness. | +| `--output` | Required | Fresh or empty artifact directory; existing nonempty directories are rejected | + +The ignored `benchmarks/fulltext-analytics/results/` directory holds generated +manifests, child logs, results and comparisons, including failure evidence. Choose +a new run subdirectory each time. Custom output paths are supported; keep them +outside the checkout or add a suitable ignore rule before running. Do not commit +generated output or test databases. + +Compare throughput, API latency, CPU, event-loop delay, RSS and exported/persisted +rows with identical settings on the same host. Both profiles use fulltext 0.9.0; +this does not measure the engine upgrade's always-on accounting cost. Histograms +and snapshot-byte observations remain off. Short runs report +`qualificationSufficient: false`; full-cadence runs require at least 60000 ms and +one collection interval plus 1000 ms. Even sufficient runs are finite local +evidence, not production or sustained-memory qualification. This benchmark is not +part of CI or the trend dashboard. + +--- + ## ST-2 — Indexed-write throughput (`indexed-write/`) Measures write ops/sec on three table variants: From 44208eb4a92e42bc9c6ee8f09536197fc0e53d23 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 08:58:39 -0700 Subject: [PATCH 11/26] fix: preserve telemetry through shutdown and configuration replacement --- .github/workflows/unit-test.yml | 7 + config/DESIGN.md | 2 +- config/RootConfigWatcher.ts | 30 +- config/fullTextTelemetryConfiguration.ts | 14 - dependencies.md | 22 +- package-lock.json | 276 ------------------ package.json | 6 +- renovate.json | 5 +- resources/FULL_TEXT.md | 6 +- resources/analytics/DESIGN.md | 6 +- resources/analytics/fullTextTelemetry.ts | 25 +- .../analytics/fullTextTelemetryRuntime.ts | 24 +- resources/analytics/write.ts | 39 ++- server/jobs/jobProcess.ts | 6 + server/threads/threadServer.ts | 2 + .../fullTextTelemetryConfiguration.test.js | 32 +- .../resources/fixtures/analyticsExitFlush.cjs | 29 ++ .../fixtures/fullTextTelemetryJobExit.cjs | 48 +++ .../fixtures/fullTextTelemetryShutdown.cjs | 77 +++++ .../fixtures/fullTextUpgradeHelpers.cjs | 10 +- .../fixtures/fullTextUpgradeProcess.cjs | 8 +- .../fulltext-upgrade/0.5.0/package-lock.json | 100 +++++++ .../fulltext-upgrade/0.5.0/package.json | 8 + .../fulltext-upgrade/0.7.0/package-lock.json | 100 +++++++ .../fulltext-upgrade/0.7.0/package.json | 8 + .../fulltext-upgrade/0.8.0/package-lock.json | 100 +++++++ .../fulltext-upgrade/0.8.0/package.json | 8 + .../installFullTextUpgradeDependencies.mjs | 36 +++ ...ativeFullTextDerivedIndexLifecycle.test.js | 1 + .../nativeFullTextIndexSchemaUpgrade.test.js | 1 + .../nativeFullTextIndexUpgradeCrash.test.js | 1 + .../resources/nativeFullTextTelemetry.test.js | 113 ++++++- .../nativeFullTextTelemetryShutdown.test.js | 216 ++++++++++++++ 33 files changed, 965 insertions(+), 401 deletions(-) create mode 100644 unitTests/resources/fixtures/analyticsExitFlush.cjs create mode 100644 unitTests/resources/fixtures/fullTextTelemetryJobExit.cjs create mode 100644 unitTests/resources/fixtures/fullTextTelemetryShutdown.cjs create mode 100644 unitTests/resources/fixtures/fulltext-upgrade/0.5.0/package-lock.json create mode 100644 unitTests/resources/fixtures/fulltext-upgrade/0.5.0/package.json create mode 100644 unitTests/resources/fixtures/fulltext-upgrade/0.7.0/package-lock.json create mode 100644 unitTests/resources/fixtures/fulltext-upgrade/0.7.0/package.json create mode 100644 unitTests/resources/fixtures/fulltext-upgrade/0.8.0/package-lock.json create mode 100644 unitTests/resources/fixtures/fulltext-upgrade/0.8.0/package.json create mode 100644 unitTests/resources/fixtures/installFullTextUpgradeDependencies.mjs create mode 100644 unitTests/resources/nativeFullTextTelemetryShutdown.test.js diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index d0013edba6..02873dbfd7 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -305,6 +305,8 @@ jobs: name: Full-text native plane (${{ matrix.os }}, Node.js v26) runs-on: ${{ matrix.os }} timeout-minutes: 15 + env: + HARPER_FULLTEXT_UPGRADE_ROOT: ${{ runner.temp }}/fulltext-upgrade strategy: fail-fast: false matrix: @@ -322,6 +324,10 @@ jobs: - name: Install dependencies run: npm install --ignore-scripts + - name: Install isolated historical full-text engines + timeout-minutes: 3 + run: npm run test:fulltext:install-previous -- "${{ runner.temp }}/fulltext-upgrade" + - name: Build run: npm run build @@ -334,6 +340,7 @@ jobs: unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js unitTests/resources/nativeFullTextTelemetry.test.js unitTests/resources/nativeFullTextTelemetryClose.test.js unitTests/resources/nativeFullTextTelemetryWorkers.test.js + unitTests/resources/nativeFullTextTelemetryShutdown.test.js unitTests/resources/nativeFullTextTableSearch.test.js unitTests/resources/nativeFullTextIndexSchemaUpgrade.test.js unitTests/resources/nativeFullTextIndexUpgradeWorkers.test.js diff --git a/config/DESIGN.md b/config/DESIGN.md index 8e3e243f34..8604f7235c 100644 --- a/config/DESIGN.md +++ b/config/DESIGN.md @@ -240,7 +240,7 @@ off on the very boot that could not read its configuration. ## `RootConfigWatcher` schedules a trailing verify-read (`config/RootConfigWatcher.ts`) -Chokidar throttles change events for each path for 50ms without a trailing emit, so closely spaced atomic renames can go unseen. After a successful read, `RootConfigWatcher` rearms a 100ms verify-read (`#read(true)`); close, terminal failure or `ConfigReadRetry` clears it. Unchanged raw text skips parsing; parsed changes use the immutable `#lastParsedConfig` so consumer mutations cannot trigger duplicate events (`rootConfigWatcher.test.js`). Root-config `OptionsWatcher`s lack this verification. Unlink emits contained `unavailable`; the first recovery emits `available` even for unchanged text. Telemetry reapplies environment overlays without file policy and retires overrides where effective values change; recreation cannot restore retired overrides. Invalid recreation retains valid policy (`fullTextTelemetryConfiguration.test.js`). +Chokidar throttles change events for each path for 50ms without a trailing emit, so closely spaced atomic renames can go unseen. After a successful read, `RootConfigWatcher` rearms a 100ms verify-read (`#read(true)`); close, terminal failure or `ConfigReadRetry` clears it. Unchanged raw text skips parsing; parsed changes use the immutable `#lastParsedConfig` so consumer mutations cannot trigger duplicate events (`rootConfigWatcher.test.js`). Root-config `OptionsWatcher`s lack this verification. Telemetry retains the last valid policy and overrides while the root file is absent or invalid. Only valid file edits retire changed overrides or disable collection (`fullTextTelemetryConfiguration.test.js`); unchanged recreation needs no availability event. ## `RootConfigWatcher` polls under Bun (`config/RootConfigWatcher.ts`) diff --git a/config/RootConfigWatcher.ts b/config/RootConfigWatcher.ts index c3b05f9dcf..d409e2bfbe 100644 --- a/config/RootConfigWatcher.ts +++ b/config/RootConfigWatcher.ts @@ -64,7 +64,6 @@ export class RootConfigWatcher extends EventEmitter { // opens it without claiming the watch is armed, so the arming re-read still runs afterwards. #barrierOpen: boolean = false; #configLoaded: boolean = false; - #unavailable: boolean = false; #readyStaged: boolean = false; #readyEmitted: boolean = false; ready: Promise; @@ -89,15 +88,6 @@ export class RootConfigWatcher extends EventEmitter { }) .on('add', this.handleChange.bind(this)) .on('change', this.handleChange.bind(this)) - .on('unlink', () => { - if (this.#closed) return; - try { - this.#unavailable = true; - this.emit('unavailable'); - } catch (error) { - logger().warn('A Harper configuration availability listener failed', errorForLog(error)); - } - }) .on('error', this.handleError.bind(this)) // Generation-bound: `#armGate.reset()` runs before the failed watcher is closed, so a // `ready` still queued on it would arm the gate on the replacement's behalf and the @@ -229,20 +219,7 @@ export class RootConfigWatcher extends EventEmitter { } handleChange() { - if (this.#read(true)) { - this.#scheduleTrailingRead(); - this.#markAvailable(); - } - } - - #markAvailable() { - if (!this.#unavailable || this.#closed) return; - this.#unavailable = false; - try { - this.emit('available'); - } catch (error) { - logger().warn('A Harper configuration availability listener failed', errorForLog(error)); - } + if (this.#read(true)) this.#scheduleTrailingRead(); } #scheduleTrailingRead() { @@ -268,10 +245,7 @@ export class RootConfigWatcher extends EventEmitter { #schedule(): boolean { this.#clearTrailingRead(); return this.#readRetry.schedule(() => { - if (this.#read(false)) { - this.#scheduleTrailingRead(); - this.#markAvailable(); - } + if (this.#read(false)) this.#scheduleTrailingRead(); }, !this.#readyEmitted); } diff --git a/config/fullTextTelemetryConfiguration.ts b/config/fullTextTelemetryConfiguration.ts index 6ec9be3317..fa30250991 100644 --- a/config/fullTextTelemetryConfiguration.ts +++ b/config/fullTextTelemetryConfiguration.ts @@ -38,26 +38,12 @@ export function watchFullTextTelemetryConfiguration( report(); } } - function unavailable() { - if (stopped) return; - try { - const root = structuredClone(overlayRootEnvConfig({}) ?? {}); - applyPolicy(refreshConfigSubtree(PARAM, previousRoot, root, resolveFullTextTelemetryPolicy)); - previousRoot = root; - } catch { - report(); - } - } function stop() { stopped = true; watcher.off('change', changed); - watcher.off('unavailable', unavailable); - watcher.off('available', changed); watcher.off('close', stop); } watcher.on('change', changed); - watcher.on('unavailable', unavailable); - watcher.on('available', changed); watcher.on('close', stop); void watcher.ready.then(changed).catch(report); return stop; diff --git a/dependencies.md b/dependencies.md index d51a471d9d..32a5a3c532 100644 --- a/dependencies.md +++ b/dependencies.md @@ -299,21 +299,9 @@ This is the inverse of the entries below — a dependency we take deliberate ste - Can be deferred: Yes. The module is not loaded for tables without full-text indexes. - Eventual removal: Remove full-text declarations and their local derived-index files, then remove the optional dependency and Harper adapter. Source records remain authoritative and unaffected. -## @harperfast/fulltext-previous (dev dependency) +## Historical full-text upgrade engines (isolated test fixtures) -- Need for usage: Exact npm alias for fulltext 0.5.0 (the version shipped in Harper 5.3.4), used only by the child-process upgrade fixture to write genuine old-engine indexes and verify downgrade refusal. Simulating an error or editing identity bytes would not exercise the actual compatibility boundary. -- Size/memory cost: One additional root package and its supported platform prebuild in development/CI. The child exits before the current engine inspects its files; production installs with dev dependencies omitted do not include the alias. -- Security/overlap: Same first-party package, exact-pinned to the prior release. Production code must not import this alias or relax its current native ABI validation to accommodate it. -- Eventual removal: Remove the alias and its Renovate exclusion with the cross-version fixtures when migration from 0.5.x is no longer supported. Renovate leaves this historical fixture fixed; do not update it together with the current wrapper pin. - -## @harperfast/fulltext-v07 (dev dependency) - -- Need for usage: Exact npm alias for fulltext 0.7.0, used by the existing isolated upgrade fixture to write a real text-only checkpoint and verify 0.8 reuse and subsequent 0.7 reopen. The separate 0.5.0 alias retains coverage of the earlier incompatible engine. -- Updates and removal: Renovate leaves this historical alias fixed. Remove it and its exclusion only when the 0.7 migration fixture is retired. -- Cost and boundary: One additional root package and supported platform prebuild in development/CI only. No production runtime import; the previous-engine child exits before the current engine opens its files. - -## @harperfast/fulltext-v08 (dev dependency) - -- Need for usage: Exact npm alias for fulltext 0.8.0. The isolated migration fixture writes a genuine prior-release checkpoint, verifies current reuse without rebuilding, advances it and checks the prior release can reopen it. -- Cost and boundary: One additional root package and supported platform prebuild in development/CI only; never imported by production code. -- Updates and removal: Keep the historical alias exact-pinned and excluded from Renovate updates. Remove it with its fixture when migration from 0.8 is no longer supported. +- Need for usage: Released fulltext 0.5.0, 0.7.0 and 0.8.0 write genuine old checkpoints; the current engine must rebuild incompatible indexes or reuse compatible ones, and prior engines verify safe reopen or downgrade refusal. Source-record IDs and checkpoints are generated during the tests, so static index files do not replace these checks. +- Installation boundary: Harper's normal dependency tree contains only the current optional engine. Separate exact-pinned manifests and integrity-checked lockfiles live under `unitTests/resources/fixtures/fulltext-upgrade/`; the dedicated native CI job provisions their packages outside the checkout. The previous-engine child exits before the current engine accesses the same index. Production code never imports historical engines or relaxes current ABI validation. +- Local qualification: Run `npm run test:fulltext:install-previous -- /absolute/scratch/fulltext-upgrade`, then set `HARPER_FULLTEXT_UPGRADE_ROOT` to that directory when running the native lifecycle, schema-upgrade and crash-recovery suites. Ordinary resource tests skip historical migration cases when the variable is absent; an explicit but invalid installation fails. CI supplies it and forbids pending tests. +- Cost and removal: The dedicated job installs three root packages and their supported platform prebuilds. Other development and CI installs incur no historical-package cost. Renovate leaves fixture versions fixed; remove each manifest, lockfile and matching coverage when migration from that version is no longer supported. diff --git a/package-lock.json b/package-lock.json index 5189d86d6f..244db0c27b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -100,9 +100,6 @@ "@aws-sdk/client-s3": "^3.1012.0", "@aws-sdk/lib-storage": "3.1156.0", "@harperfast/code-guidelines": "^0.1.2", - "@harperfast/fulltext-previous": "npm:@harperfast/fulltext@0.5.0", - "@harperfast/fulltext-v07": "npm:@harperfast/fulltext@0.7.0", - "@harperfast/fulltext-v08": "npm:@harperfast/fulltext@0.8.0", "@harperfast/integration-testing": "^1.0.0", "@modelcontextprotocol/sdk": "^1.29.0", "@types/busboy": "^1.5.4", @@ -2481,279 +2478,6 @@ "node": "^22.18.0 || >=24.0.0" } }, - "node_modules/@harperfast/fulltext-previous": { - "name": "@harperfast/fulltext", - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext/-/fulltext-0.5.0.tgz", - "integrity": "sha512-rH2Q3LJLHPhxi8muzPLg00STbO7RFIqAgLA3tpBzSRdsJVuCV9ZMO2StlSZnUAPMnF5wF9SieuhHT9wi1vINBQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^22.18.0 || >=24.0.0" - }, - "optionalDependencies": { - "@harperfast/fulltext-darwin-arm64": "0.5.0", - "@harperfast/fulltext-linux-arm64-gnu": "0.5.0", - "@harperfast/fulltext-linux-x64-gnu": "0.5.0", - "@harperfast/fulltext-win32-x64-msvc": "0.5.0" - } - }, - "node_modules/@harperfast/fulltext-previous/node_modules/@harperfast/fulltext-darwin-arm64": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-darwin-arm64/-/fulltext-darwin-arm64-0.5.0.tgz", - "integrity": "sha512-0OtKonopmXaP0/9wvpCuo/sklNxOEiQWnVf7aUsthH0B3sKUYT7HGnGtYkzqqUxdsFNlHPBr3IShyrWx9nAWYQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/@harperfast/fulltext-previous/node_modules/@harperfast/fulltext-linux-arm64-gnu": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-arm64-gnu/-/fulltext-linux-arm64-gnu-0.5.0.tgz", - "integrity": "sha512-QfNlYUntkal56jZejQ8cbSj8qvUlYv21V2j9xVctkFv+gmfn892kdU4oCg7dR7Qift8/huKUOyL/eGO0qYbgGw==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/@harperfast/fulltext-previous/node_modules/@harperfast/fulltext-linux-x64-gnu": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-x64-gnu/-/fulltext-linux-x64-gnu-0.5.0.tgz", - "integrity": "sha512-+g5LMXhtODen+pGgmyTxcCuUAlk2NTeFOvr6duqUBVx542Hd/uDqPC9tUMjL12LJ9ci3LFqGwgW3FkxGedT7KA==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/@harperfast/fulltext-previous/node_modules/@harperfast/fulltext-win32-x64-msvc": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-win32-x64-msvc/-/fulltext-win32-x64-msvc-0.5.0.tgz", - "integrity": "sha512-PseSz47BMcIOMQdYUFoAUqAEyeHzceX+QQFun0wVCu/PtERPvtPWzzZxV4ofySzz+c9FkJHTSDu4XbAZhkaqkA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/@harperfast/fulltext-v07": { - "name": "@harperfast/fulltext", - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext/-/fulltext-0.7.0.tgz", - "integrity": "sha512-elDKwjn1PYRkH3Fpc4pIC/YxgfAUfHECtisyVNBEiK1XQArcKQUYxVUJ1KKsxrcn5qzf2vlYM71SyONlAm5zQQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^22.18.0 || >=24.0.0" - }, - "optionalDependencies": { - "@harperfast/fulltext-darwin-arm64": "0.7.0", - "@harperfast/fulltext-linux-arm64-gnu": "0.7.0", - "@harperfast/fulltext-linux-x64-gnu": "0.7.0", - "@harperfast/fulltext-win32-x64-msvc": "0.7.0" - } - }, - "node_modules/@harperfast/fulltext-v07/node_modules/@harperfast/fulltext-darwin-arm64": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-darwin-arm64/-/fulltext-darwin-arm64-0.7.0.tgz", - "integrity": "sha512-oamX0LDMKt5o1PXDk0T4SN92Z1OvlmnBWVknSExr4SDqmUit9MonSkVvGcTRQL/fAVHnI7kwlNqpsDAywxOE3Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/@harperfast/fulltext-v07/node_modules/@harperfast/fulltext-linux-arm64-gnu": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-arm64-gnu/-/fulltext-linux-arm64-gnu-0.7.0.tgz", - "integrity": "sha512-jH8szxy+9Q84/O/J7cRo6GRHIntO73PyzdD0Gx9uX/6fsPLJ0+Ldd0+JFok2ygd8Mv72pmI0pcy0UI7W28vi5w==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/@harperfast/fulltext-v07/node_modules/@harperfast/fulltext-linux-x64-gnu": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-x64-gnu/-/fulltext-linux-x64-gnu-0.7.0.tgz", - "integrity": "sha512-36xlr2jZa5VmAeUIcPMTs1MRRpvJ20P9RfqjSu75NLD/Zr7J5o71E4b8IkX3R0qvaya1P54BjNTrmQc8nd/amA==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/@harperfast/fulltext-v07/node_modules/@harperfast/fulltext-win32-x64-msvc": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-win32-x64-msvc/-/fulltext-win32-x64-msvc-0.7.0.tgz", - "integrity": "sha512-J2Wfs0w/kYlNaUJvwifMPkWWmfRG3cZme/Ha7ZGLO9zMzVsefLiEDYOHXNPrmJfKW+TebSEAu6Z0NB7H63ix4A==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/@harperfast/fulltext-v08": { - "name": "@harperfast/fulltext", - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext/-/fulltext-0.8.0.tgz", - "integrity": "sha512-7T+i2PbdsyrXwb4V2mcE23Pk8DwOl12vgE8YodTe/KUeS7tzJp8I/uQOkD5YiW1ga4WasfLVbZrcTi5pULMjAg==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^22.18.0 || >=24.0.0" - }, - "optionalDependencies": { - "@harperfast/fulltext-darwin-arm64": "0.8.0", - "@harperfast/fulltext-linux-arm64-gnu": "0.8.0", - "@harperfast/fulltext-linux-x64-gnu": "0.8.0", - "@harperfast/fulltext-win32-x64-msvc": "0.8.0" - } - }, - "node_modules/@harperfast/fulltext-v08/node_modules/@harperfast/fulltext-darwin-arm64": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-darwin-arm64/-/fulltext-darwin-arm64-0.8.0.tgz", - "integrity": "sha512-Egmt0FsOqgTEeZo6YpxMbywTbaO8aH3gDqHpC+lfmrqXAFiDub8NqQ72xSAI0VUfSMXUL83/NoHGX68brZ+upA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/@harperfast/fulltext-v08/node_modules/@harperfast/fulltext-linux-arm64-gnu": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-arm64-gnu/-/fulltext-linux-arm64-gnu-0.8.0.tgz", - "integrity": "sha512-YfLa+bjnZUXI5bY3JPOmlkn3DlKXA3zheryUxD+IKTjUvR5ekNoYbl5WTEzYhqRZGZs2o07ct2+Zb3otiUP3Ww==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/@harperfast/fulltext-v08/node_modules/@harperfast/fulltext-linux-x64-gnu": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-x64-gnu/-/fulltext-linux-x64-gnu-0.8.0.tgz", - "integrity": "sha512-xLTpOAyNax0FpKKoMsnIRgcyMD1sdCVhKmBXO5pMPB9vHoAoTBoeyGjKxAv2HGqwcOXT5sfhAGmCUpqPlbDtVA==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/@harperfast/fulltext-v08/node_modules/@harperfast/fulltext-win32-x64-msvc": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-win32-x64-msvc/-/fulltext-win32-x64-msvc-0.8.0.tgz", - "integrity": "sha512-GKQ9GR8WuN8T4tqZS+smQanLm97lMiA2uZv6pMJk79zh4xtNrC5wOzARx3/J3l3wclI2HCf+iVKgn1OxOfvR7w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, "node_modules/@harperfast/fulltext-win32-x64-msvc": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/@harperfast/fulltext-win32-x64-msvc/-/fulltext-win32-x64-msvc-0.9.0.tgz", diff --git a/package.json b/package.json index 4252ec2a76..bfe5deab1a 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,8 @@ "test:unit:typestrip:resources:core:a-n": "node unitTests/runTypeStripTests.mts --source-only \"unitTests/resources/**/[a-nA-N]*.js\" --exclude \"unitTests/resources/**/*.bench.js\" --exclude \"unitTests/resources/**/*{[Ii]ndex,[Hh]nsw,[Vv]ector}*.js\"", "test:unit:typestrip:resources:core:o-z": "node unitTests/runTypeStripTests.mts --source-only \"unitTests/resources/**/*.js\" --exclude \"unitTests/resources/**/*.bench.js\" --exclude \"unitTests/resources/**/*{[Ii]ndex,[Hh]nsw,[Vv]ector}*.js\" --exclude \"unitTests/resources/**/[a-nA-N]*.js\"", "test:unit:typestrip:resources:indexes": "node unitTests/runTypeStripTests.mts --source-only \"unitTests/resources/**/*{[Ii]ndex,[Hh]nsw,[Vv]ector}*.js\" --exclude \"unitTests/resources/**/*.bench.js\"", - "test:types:unit": "tsc --project unitTests/tsconfig.harness.json" + "test:types:unit": "tsc --project unitTests/tsconfig.harness.json", + "test:fulltext:install-previous": "node unitTests/resources/fixtures/installFullTextUpgradeDependencies.mjs" }, "imports": { "#src/*.js": { @@ -162,9 +163,6 @@ "@aws-sdk/client-s3": "^3.1012.0", "@aws-sdk/lib-storage": "3.1156.0", "@harperfast/code-guidelines": "^0.1.2", - "@harperfast/fulltext-previous": "npm:@harperfast/fulltext@0.5.0", - "@harperfast/fulltext-v07": "npm:@harperfast/fulltext@0.7.0", - "@harperfast/fulltext-v08": "npm:@harperfast/fulltext@0.8.0", "@harperfast/integration-testing": "^1.0.0", "@modelcontextprotocol/sdk": "^1.29.0", "@types/busboy": "^1.5.4", diff --git a/renovate.json b/renovate.json index 4ec897a830..a350c885d7 100644 --- a/renovate.json +++ b/renovate.json @@ -53,10 +53,9 @@ "matchPackageNames": ["!alasql"] }, { - "description": "Keep the historical fulltext 0.5.0, 0.7.0 and 0.8.0 upgrade fixtures fixed; remove this rule with the fixture alias.", + "description": "Keep the separately provisioned historical fulltext upgrade engines exact-pinned; remove each fixture when its migration support ends.", "matchManagers": ["npm"], - "matchDepTypes": ["devDependencies"], - "matchDepNames": ["@harperfast/fulltext-previous", "@harperfast/fulltext-v07", "@harperfast/fulltext-v08"], + "matchFileNames": ["unitTests/resources/fixtures/fulltext-upgrade/**"], "enabled": false } ] diff --git a/resources/FULL_TEXT.md b/resources/FULL_TEXT.md index da2e67ea04..af622a8486 100644 --- a/resources/FULL_TEXT.md +++ b/resources/FULL_TEXT.md @@ -207,7 +207,7 @@ analytics: The implied periodic interval is 60,000 milliseconds per handle. Calls are staggered across existing writer/readers, and their rows use Harper's existing analytics flush, storage, retention and protected `get_analytics` interface. Successful handle closes also collect final counters, so the interval is not a limit on total snapshot or export frequency. Collection never opens a cold index. To tune it, set `intervalMilliseconds` between 1,000 and 2,147,483,647; synchronous snapshots and export work have measurable cost, so qualify the intended number of indexes, reader churn and cadence. No production overhead bound is claimed. -Set `enabled: false` or remove the block to stop scheduled polling and persistent full-text export. A synchronous observation already executing completes. Editing unrelated settings does not restart collection. Invalid telemetry edits retain the last valid policy. Deleting the configuration file removes its telemetry settings. An in-process override is retired if deletion changes its matching configuration value after environment overlays are applied; recreating the file does not restore retired overrides. Environment overlays still apply while the file is missing. Disabling analytics also disables its full-text collection demand. Internal registered consumers can independently request observations; removing the last demand stops the timer. These are implementation hooks, not a public SDK, and currently have no production caller. Consumer-only observations do not enable persistent analytics. With operator export enabled, persisted fields reflect the union of operator and consumer recording requests. +Set `enabled: false` or remove the block to stop scheduled polling and persistent full-text export. A synchronous observation already executing completes. Editing unrelated settings does not restart collection. Invalid edits or a missing configuration file retain the last valid telemetry policy and in-process overrides. Recreating an unchanged file preserves them. To disable collection, write a valid file with `enabled: false` or remove the telemetry block; valid edits retire overrides only when their matching effective value changes after environment overlays. Disabling analytics also disables its full-text collection demand. Internal registered consumers can independently request observations; removing the last demand stops the timer. These are implementation hooks, not a public SDK, and currently have no production caller. Consumer-only observations do not enable persistent analytics. With operator export enabled, persisted fields reflect the union of operator and consumer recording requests. Workers and the receiving main thread enforce their current export policy independently. During configuration transitions, the receiver can discard worker rows, including a final close observation; these policy-filtered rows are not counted in `droppedRows` and may have no later replacement. @@ -215,8 +215,8 @@ Workers and the receiving main thread enforce their current export policy indepe Read `get_analytics` with `metric: fulltext`. Each stored row identifies `path` (`.
/` with percent-encoded components, including literal dots as `%2E`, or `index:` when a component cannot be percent-encoded or the encoded label exceeds 256 bytes), `method` (`writer` or `reader`), worker `thread`, a bounded-lifetime `series` reset marker and actual `sampledAt`. Field names flatten the wrapper snapshot: for example, `native_operations_search_completed`, `native_operations_search_queue_sumNanoseconds`, `native_queues_ordinarySearch_commands` and `logicalBatches_completed`. Values are flat JSON scalars; exact bigint counters, nanoseconds, bytes, histogram counts and bounds are decimal strings. Availability/pending/error flags distinguish missing observations from zero. Null can mean an unavailable observation, an empty queue's oldest age, or a histogram's unbounded final bucket. Histograms retain noncumulative buckets; no percentiles are invented. Histogram deltas cover only intervals with recording enabled, not an entire collection gap when recording was disabled. -Native uint64 counter wraps are unwrapped into exact logical-series totals. Reader reopen and overlapping retired readers carry counters into the same worker/index/role series; document/segment/generation observations come from the most recently observed eligible reader rather than summing duplicate snapshots. Queue occupancy sums live handles of the same role; oldest age uses their maximum. Shared queue capacities overlap: never add ordinary/expensive capacities or treat them as a process-wide budget. Final close counters are sampled once after settlement; no recurring scrape runs while close is in flight. +Native uint64 counter wraps are unwrapped into exact logical-series totals. Reader reopen and overlapping retired readers carry counters into the same worker/index/role series; document/segment/generation observations come from the most recently observed eligible reader rather than summing duplicate snapshots. Queue occupancy sums live handles of the same role; oldest age uses their maximum. Shared queue capacities overlap: never add ordinary/expensive capacities or treat them as a process-wide budget. Final close counters are sampled once after settlement; no recurring scrape runs while close is in flight. Graceful HTTP/job-worker shutdown flushes pending rows and final observations after owner cleanup, including one last observation of still-open handles. Forced termination and receiver-side policy filtering can still lose reports. -Rollups keep the newest cumulative row per worker/index/role instead of averaging counters. For rates, decode decimal counters with `BigInt`, require the same `series` and complete/available observations, subtract exact totals and divide by actual `sampledAt` elapsed time. Check integer safety before converting a delta to `number`. A new series, unavailable samples or incomplete coverage require a new baseline; never report a missing interval as zero activity. `complete: false` flags lost final handle counts. The next successful live observation starts a new series so later rates can use a new baseline; retained absolute totals need not reset to zero. Dropping a pending observation does not invalidate cumulative totals from active handles; `observationAgeMilliseconds` distinguishes retained observations from the latest attempt. Pending rows are coalesced per logical series in a 128-row buffer; overflow drops the oldest pending row and increments exact `droppedRows`. Historical rows follow configured analytics retention, not handle lifetime. +Rollups keep the newest cumulative row per worker/index/role instead of averaging counters. For rates, decode decimal counters with `BigInt`, require the same `series` and complete/available observations, subtract exact totals and divide by actual `sampledAt` elapsed time. Check integer safety before converting a delta to `number`. A new series, unavailable samples or incomplete coverage require a new baseline; never report a missing interval as zero activity. `complete: false` flags lost final handle counts. The next successful live observation starts a new series so later rates can use a new baseline; retained absolute totals need not reset to zero. Dropping a pending observation does not invalidate cumulative totals from active handles; `observationAgeMilliseconds` distinguishes retained observations from the latest attempt. Pending rows are coalesced per logical series in a 128-row buffer; overflow prefers the least recently refreshed pending row from a still-live group before a final row, and increments exact `droppedRows`. Historical rows follow configured analytics retention, not handle lifetime. Do not add facade logical batches to their native frame counts, sum overlapping legacy/native timing spans, or sum reader document counts across workers. Generation IDs are not source-log positions or freshness timestamps. Harper still owns readiness, source catch-up, authorization, hydration and end-to-end request attribution; this telemetry does not replace those checks. Queries, terms, record IDs, users, raw cursors, filesystem paths and raw exception messages are not exported labels. Detailed automatic merge statistics, cancellation and per-index RSS remain explicitly unsupported. diff --git a/resources/analytics/DESIGN.md b/resources/analytics/DESIGN.md index 8a064f3fd3..26ab274a5f 100644 --- a/resources/analytics/DESIGN.md +++ b/resources/analytics/DESIGN.md @@ -73,8 +73,8 @@ always a caller's measure. ## Full-text collection follows actual handle ownership -`fullTextTelemetry.ts` keeps private per-handle baselines and carries exact counters into a stable worker/index/role series. `nativeFullTextTelemetry.test.js` pins reader churn, wrap/precision, optional recording unions and peer-gauge aggregation; cached aggregate names include null leaves, so a live peer's queue age survives a newest null observation. `nativeFullTextTelemetryClose.test.js` holds backend close across live-peer timer ticks and recording changes: closing handles are neither sampled nor reconfigured until settlement. Completed registrations release references; inactive groups remain until their final row drains. Pending export has a 128-row bound with explicit drop counts. +`fullTextTelemetry.ts` keeps private per-handle baselines and carries exact counters into a stable worker/index/role series. `nativeFullTextTelemetry.test.js` pins reader churn, wrap/precision, optional recording unions and peer-gauge aggregation; cached aggregate names include null leaves, so a live peer's queue age survives a newest null observation. `nativeFullTextTelemetryClose.test.js` holds backend close across live-peer timer ticks and recording changes: closing handles are neither sampled nor reconfigured until settlement. Completed registrations release references; inactive groups remain until their final row drains. Pending export has a 128-row bound with explicit drop counts. Graceful HTTP/job exit reuses the serialized analytics flush after owner cleanup, drains pending rows and samples open handles without overflowing that bound (`nativeFullTextTelemetryShutdown.test.js`). -`fullTextTelemetryRuntime.ts` joins removable internal consumer demand with live operator policy. The consumer hook is an internal diagnostic/test seam with no production caller yet. Only operator demand enables persistence; persisted optional fields reflect the union of recording requests. No demand means no timer; no cold handles open. Exact counters are decimal strings; `cumulative` retains the newest worker row instead of averaging. Lost final counts force a new series on the next live observation. FIFO pending-row drops preserve totals only while a group remains live. Main also filters worker rows by its receive-time policy; transition losses, including final rows, do not increment `droppedRows`. Failed writer closes resume collection while the engine remains installed. `aggregationCycle.test.js` verifies counters above 2^53 survive storage and `get_analytics`. +`fullTextTelemetryRuntime.ts` joins removable internal consumer demand with live operator policy. The consumer hook is an internal diagnostic/test seam with no production caller yet. Only operator demand enables persistence; persisted optional fields reflect the union of recording requests. No demand means no timer; no cold handles open. Exact counters are decimal strings; `cumulative` retains the newest worker row instead of averaging. Lost final counts force a new series on the next live observation. Overflow prefers replaceable live rows, ordered by last refresh, before final rows; dropped live totals remain recoverable. Main also filters worker rows by its receive-time policy; transition losses, including final rows, do not increment `droppedRows`. Failed writer closes resume collection while the engine remains installed. `aggregationCycle.test.js` verifies counters above 2^53 survive storage and `get_analytics`. -`watchFullTextTelemetryConfiguration` initializes and reloads through the shared watcher's nested root, environment overlay and transactional `refreshConfigSubtree`; it never reads the missing camel-case flat aggregate. Invalid edits retain valid policy. Deletion retires overrides only where matching values change after environment overlays; recreation does not restore them. `fullTextTelemetryConfiguration.test.js` pins initialization, override retirement and recovery. Native runtime/query policy remains frozen separately. Binding telemetry types are structural: `buildDist.test.mjs` checks compilation and emitted declarations without the optional package. +`watchFullTextTelemetryConfiguration` initializes and reloads through the shared watcher's nested root, environment overlay and transactional `refreshConfigSubtree`; it never reads the missing camel-case flat aggregate. Invalid edits retain valid policy. Missing files retain valid policy and overrides, matching the root watcher; valid edits retire only changed overrides. `fullTextTelemetryConfiguration.test.js` pins initialization, override retirement and deletion/recreation. Native runtime/query policy remains frozen separately. Binding telemetry types are structural: `buildDist.test.mjs` checks compilation and emitted declarations without the optional package. diff --git a/resources/analytics/fullTextTelemetry.ts b/resources/analytics/fullTextTelemetry.ts index 76473f2f6f..3114cdccf8 100644 --- a/resources/analytics/fullTextTelemetry.ts +++ b/resources/analytics/fullTextTelemetry.ts @@ -82,7 +82,7 @@ type Consumer = { }; const UINT64_MODULUS = 1n << 64n; -export function fullTextNativeCounterDelta(current: bigint, previous: bigint): bigint { +function fullTextNativeCounterDelta(current: bigint, previous: bigint): bigint { return current >= previous ? current - previous : UINT64_MODULUS - previous + current; } @@ -92,7 +92,7 @@ function isCounter(path: string[]): boolean { return path[0] === 'native' && (path[1] === 'phases' || (path[1] === 'operations' && path[3] !== 'dispatched')); } -export function flattenFullTextTelemetry(snapshot: FullTextTelemetrySnapshot): FlatSnapshot { +function flattenFullTextTelemetry(snapshot: FullTextTelemetrySnapshot): FlatSnapshot { const counters = new Map(); const gauges: FullTextTelemetryRow = {}; const aggregateGauges = new Map(); @@ -296,6 +296,17 @@ export class FullTextTelemetryCollector { return rows; } + collectFinalRows(): FullTextTelemetryRow[] { + const rows = this.takeRows(); + if (this.collecting) + for (const entry of [...this.#entries]) { + if (entry.closing) continue; + this.#observe(entry); + rows.push(...this.takeRows()); + } + return rows; + } + stop(): void { if (this.#stopped) return; this.#operator = { ...DEFAULT_FULL_TEXT_TELEMETRY_POLICY }; @@ -446,8 +457,14 @@ export class FullTextTelemetryCollector { } row[name] = observed ? total.toString() : null; } - if (!this.#pending.has(entry.group) && this.#pending.size >= this.#maxPendingRows) { - const oldest = this.#pending.keys().next().value!; + if (this.#pending.has(entry.group)) this.#pending.delete(entry.group); + else if (this.#pending.size >= this.#maxPendingRows) { + let oldest = this.#pending.keys().next().value!; + for (const candidate of this.#pending.keys()) + if (candidate.entries.size) { + oldest = candidate; + break; + } this.#pending.delete(oldest); this.#droppedRows++; row.droppedRows = this.#droppedRows.toString(); diff --git a/resources/analytics/fullTextTelemetryRuntime.ts b/resources/analytics/fullTextTelemetryRuntime.ts index 9477e06f8d..2947c3a5f6 100644 --- a/resources/analytics/fullTextTelemetryRuntime.ts +++ b/resources/analytics/fullTextTelemetryRuntime.ts @@ -1,6 +1,5 @@ import { createHash } from 'node:crypto'; import { onStartup } from '../../utility/lifecycle.ts'; -import { registerShutdownDrain } from '../../components/shutdownDrain.ts'; import { loggerWithTag } from '../../utility/logging/logger.ts'; import { watchFullTextTelemetryConfiguration } from '../../config/fullTextTelemetryConfiguration.ts'; import { addAnalyticsListener, requestAnalyticsFlush, checkAnalyticsEnabled } from './write.ts'; @@ -52,7 +51,20 @@ export function acceptsFullTextTelemetryRow(row: Record): boole export function initializeFullTextTelemetry(): void { if (started) return; - const listener = addAnalyticsListener((metrics) => { + const listener = addAnalyticsListener((metrics, final = false) => { + if (final) { + try { + stopWatching?.(); + if (fullTextTelemetryCollector.collecting || fullTextTelemetryCollector.hasPendingRows) { + const rows = fullTextTelemetryCollector.collectFinalRows(); + if (persistenceActive && checkAnalyticsEnabled(false)) metrics.push(...rows); + } + } finally { + removeListener?.(); + fullTextTelemetryCollector.stop(); + } + return; + } if (!persistenceActive || !checkAnalyticsEnabled(false) || !fullTextTelemetryCollector.hasPendingRows) return; const rows = fullTextTelemetryCollector.takeRows(); metrics.push(...rows); @@ -91,11 +103,3 @@ export function registerNativeFullTextTelemetry(handle: unknown, index: string, } onStartup(initializeFullTextTelemetry); -registerShutdownDrain({ - hasWork: () => false, - drain: async () => { - stopWatching?.(); - removeListener?.(); - fullTextTelemetryCollector.stop(); - }, -}); diff --git a/resources/analytics/write.ts b/resources/analytics/write.ts index 69dd477a6a..1eb84494e4 100644 --- a/resources/analytics/write.ts +++ b/resources/analytics/write.ts @@ -49,6 +49,9 @@ let analyticsEnabled = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) > -1; let analyticsReadOnlyChecked = false; let sendAnalyticsTimeout: NodeJS.Timeout; let analyticsGeneration = 0; +let analyticsExiting = false; +let runningAnalyticsFlush: Promise | undefined; +let exitAnalyticsFlush: Promise | undefined; onStartup(() => { analyticsEnabled = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) > -1; @@ -160,6 +163,7 @@ function actionKey(metric: string, path?: string, method?: string, type?: string } function recordKeyedAction(key: string, value: Value, metric: string, path?: string, method?: string, type?: string) { + if (analyticsExiting) return; const action = activeActions.get(key); if (action) { recordExistingAction(value, action); @@ -192,7 +196,23 @@ export function addAnalyticsListener(callback) { } export function requestAnalyticsFlush(): void { - if (checkAnalyticsEnabled() && !sendAnalyticsTimeout) contextStorage.exit(sendAnalytics); + if (!analyticsExiting && checkAnalyticsEnabled() && !sendAnalyticsTimeout) contextStorage.exit(sendAnalytics); +} + +/** Deliver the last window after resource owners close, without starting new periodic work. */ +export function flushAnalyticsForExit(): Promise { + if (exitAnalyticsFlush) return exitAnalyticsFlush; + analyticsExiting = true; + clearTimeout(sendAnalyticsTimeout); + sendAnalyticsTimeout = null; + return (exitAnalyticsFlush = contextStorage.exit(async () => { + try { + await runningAnalyticsFlush; + } finally { + if (checkAnalyticsEnabled(false)) await flushAnalytics(true); + else for (const listener of [...analyticsListeners]) listener([], true); + } + })); } const IDEAL_PERCENTILES = [0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99, 0.999, 1]; @@ -218,14 +238,25 @@ function sendAnalytics() { }, analyticsDelay).unref(); } -async function flushAnalytics() { +function flushAnalytics(final = false): Promise { + const run = () => collectAnalytics(final); + const result = runningAnalyticsFlush ? runningAnalyticsFlush.then(run, run) : run(); + runningAnalyticsFlush = result; + const settled = () => { + if (runningAnalyticsFlush === result) runningAnalyticsFlush = undefined; + }; + void result.then(settled, settled); + return result; +} + +async function collectAnalytics(final: boolean) { sendAnalyticsTimeout = null; const generation = analyticsGeneration; // Rotate before the first yield below: a sample recorded mid-flush would otherwise be added to // an entry already reported, and dropped with it. const reportingActions = activeActions; activeActions = new Map(); - const period = performance.now() - analyticsStart; + const period = analyticsStart ? performance.now() - analyticsStart : 0; analyticsStart = 0; // Read before the first yield too, so the event loop's samples from the rest of the flush belong // to the next report. @@ -319,7 +350,7 @@ async function flushAnalytics() { }); for (const listener of [...analyticsListeners]) { if (generation !== analyticsGeneration) return; - listener(metrics); + listener(metrics, final); } if (generation !== analyticsGeneration) return; if (parentPort) diff --git a/server/jobs/jobProcess.ts b/server/jobs/jobProcess.ts index d96fa96d1d..0e55def8b6 100644 --- a/server/jobs/jobProcess.ts +++ b/server/jobs/jobProcess.ts @@ -23,6 +23,7 @@ import { getEnvBuiltInComponents } from './../../components/Application.ts'; import { PACKAGE_ROOT } from '../../utility/packageUtils.js'; import * as env from '../../utility/environment/environmentManager.ts'; import { runStartup } from '../../utility/lifecycle.ts'; +import { flushAnalyticsForExit } from '../../resources/analytics/write.ts'; const JOB_NAME = process.env[(hdbTerms as any).PROCESS_NAME_ENV_PROP] as string; const JOB_ID = JOB_NAME.substring(4); @@ -110,6 +111,11 @@ const JOB_ID = JOB_NAME.substring(4); } catch (closeErr) { harperLogger.warn('Error releasing database handles on job worker exit:', closeErr); } + try { + await flushAnalyticsForExit(); + } catch (analyticsErr) { + harperLogger.warn('Error flushing analytics on job worker exit:', analyticsErr); + } if (databaseHandlesReleased) notifyJobCleanupComplete(); // On Bun 1.3.13, calling process.exit() in a worker thread with lmdb-js loaded // while sibling workers are running causes a NAPI fatal error crash. Unref diff --git a/server/threads/threadServer.ts b/server/threads/threadServer.ts index 105c0befe5..57e1ef5a1c 100644 --- a/server/threads/threadServer.ts +++ b/server/threads/threadServer.ts @@ -62,6 +62,7 @@ import * as globals from '../../globals.js'; import { whenScopesClosed, startDeferredScopeCloses } from '../../components/scopeShutdown.ts'; import { getComponentName } from '../../components/componentLoader.ts'; import { onStartup } from '../../utility/lifecycle.ts'; +import { flushAnalyticsForExit } from '../../resources/analytics/write.ts'; let debugThreads = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG); let inspectorInitialized = false; @@ -261,6 +262,7 @@ function shutdownServers(exitDeadline: number | undefined = getProcessExitDeadli } }) ) + .then(phase('flushing analytics', flushAnalyticsForExit)) .then(() => { if (!isMainThread) realExit(0); }); diff --git a/unitTests/config/fullTextTelemetryConfiguration.test.js b/unitTests/config/fullTextTelemetryConfiguration.test.js index 79125d41c0..f630b44e36 100644 --- a/unitTests/config/fullTextTelemetryConfiguration.test.js +++ b/unitTests/config/fullTextTelemetryConfiguration.test.js @@ -2,6 +2,7 @@ const assert = require('node:assert'); const { mkdtempSync, writeFileSync, rmSync } = require('node:fs'); const { join } = require('node:path'); const { stringify } = require('yaml'); +const { setTimeout: delay } = require('node:timers/promises'); const { RootConfigWatcher } = require('#src/config/RootConfigWatcher'); const { watchFullTextTelemetryConfiguration } = require('#src/config/fullTextTelemetryConfiguration'); const { resolveFullTextTelemetryPolicy } = require('#src/resources/analytics/fullTextTelemetry'); @@ -101,28 +102,39 @@ describe('Live full-text telemetry configuration', function () { await waitFor(() => watcher.config?.analytics.fullText.enabled === true); assert.strictEqual(samples.length, count); }); - it('disables on file deletion and resumes on recreation', async () => { + it('retains the last valid policy and overrides through deletion and identical recreation', async () => { const samples = []; + env.setProperty(PARAM + '_histograms', true); observe({ enabled: true }, samples); await waitFor(() => samples.at(-1)?.enabled === true); + const count = samples.length; rmSync(file); - await waitFor(() => samples.at(-1)?.enabled === false); + await delay(250); + assert.strictEqual(samples.at(-1).enabled, true); + assert.strictEqual(samples.length, count); + assert.strictEqual(env.getConfigOverrides()[PARAM + '_histograms'], true); + const reads = watcher._readCountForTests; writeFileSync(file, stringify(root({ enabled: true }))); - await waitFor(() => samples.at(-1)?.enabled === true); + await waitFor(() => watcher._readCountForTests > reads); + assert.strictEqual(samples.length, count); + writeFileSync(file, stringify(root({ enabled: false, histograms: false }))); + await waitFor(() => samples.at(-1)?.enabled === false); + assert.strictEqual(env.getConfigOverrides()?.[PARAM + '_histograms'], undefined); }); it('applies a recreated changed file once and ignores invalid revival policy', async () => { const samples = []; observe({ enabled: true }, samples); await waitFor(() => samples.at(-1)?.enabled === true); rmSync(file); - await waitFor(() => samples.at(-1)?.enabled === false); - const disabledCount = samples.length; + await delay(250); + assert.strictEqual(samples.at(-1).enabled, true); + const retainedCount = samples.length; writeFileSync(file, stringify(root({ enabled: 'invalid' }))); await waitFor(() => watcher.config?.analytics.fullText.enabled === 'invalid'); - assert.strictEqual(samples.length, disabledCount); + assert.strictEqual(samples.length, retainedCount); writeFileSync(file, stringify(root({ enabled: true, intervalMilliseconds: 2000 }))); - await waitFor(() => samples.at(-1)?.enabled === true); - assert.strictEqual(samples.length, disabledCount + 1); + await waitFor(() => samples.at(-1)?.intervalMilliseconds === 2000); + assert.strictEqual(samples.length, retainedCount + 1); assert.strictEqual(samples.at(-1).intervalMilliseconds, 2000); }); it('honors reasserted environment policy over edits to the file', async () => { @@ -134,10 +146,8 @@ describe('Live full-text telemetry configuration', function () { writeFileSync(file, stringify(root({ enabled: false, intervalMilliseconds: 3000 }))); await waitFor(() => watcher.config?.analytics.fullText.intervalMilliseconds === 3000); assert.strictEqual(samples.at(-1).intervalMilliseconds, 2000); - let unavailable = false; - watcher.once('unavailable', () => (unavailable = true)); rmSync(file); - await waitFor(() => unavailable); + await delay(250); assert.strictEqual(samples.at(-1).enabled, true); assert.strictEqual(samples.at(-1).intervalMilliseconds, 2000); }); diff --git a/unitTests/resources/fixtures/analyticsExitFlush.cjs b/unitTests/resources/fixtures/analyticsExitFlush.cjs new file mode 100644 index 0000000000..9bd45c9278 --- /dev/null +++ b/unitTests/resources/fixtures/analyticsExitFlush.cjs @@ -0,0 +1,29 @@ +'use strict'; + +const assert = require('node:assert'); +const { parentPort } = require('node:worker_threads'); +const env = require('#src/utility/environment/environmentManager'); +env.initSync(); +const analytics = require('#src/resources/analytics/write'); +parentPort.ref(); +const order = []; +analytics.setAnalyticsEnabled(true); +analytics.addAnalyticsListener((_metrics, final) => order.push(final ? 'final-delivered' : 'ordinary-delivered')); +analytics.recordAction(() => { + order.push('ordinary-started'); + queueMicrotask(() => { + order.push('exit-requested'); + const completed = analytics.flushAnalyticsForExit(); + assert.strictEqual(analytics.flushAnalyticsForExit(), completed); + completed.then( + () => { + parentPort.postMessage({ type: 'flush-completed', order }); + parentPort.close(); + }, + (error) => { + throw error; + } + ); + }); + return { count: 1 }; +}, 'shutdown-overlap'); diff --git a/unitTests/resources/fixtures/fullTextTelemetryJobExit.cjs b/unitTests/resources/fixtures/fullTextTelemetryJobExit.cjs new file mode 100644 index 0000000000..073ab56a6b --- /dev/null +++ b/unitTests/resources/fixtures/fullTextTelemetryJobExit.cjs @@ -0,0 +1,48 @@ +'use strict'; + +const { parentPort, workerData } = require('node:worker_threads'); +const { onStartup } = require('#src/utility/lifecycle'); + +onStartup(() => + onStartup(async () => { + const { loadFullTextNativeBinding } = require('#src/resources/indexes/fullTextNativeBinding'); + const { + initializeFullTextTelemetry, + registerNativeFullTextTelemetry, + fullTextTelemetryCollector, + } = require('#src/resources/analytics/fullTextTelemetryRuntime'); + const analytics = require('#src/resources/analytics/write'); + const { waitFor } = require('../../waitFor'); + initializeFullTextTelemetry(); + analytics.setAnalyticsEnabled(true); + await waitFor(() => fullTextTelemetryCollector.collecting, 5000); + const binding = await loadFullTextNativeBinding(); + const writer = await binding.openNativeFullTextIndex({ + path: workerData.indexPath, + indexId: workerData.indexLabel, + generation: 'job-generation', + fields: [{ name: 'title' }], + analyzer: 'english@2', + positions: true, + limits: { + indexingThreads: 2, + searchThreads: 2, + writerMemoryBytes: 64 * 1024 * 1024, + maxQueuedCommands: 16, + maxQueuedBytes: 64 * 1024 * 1024, + maxBatchBytes: 8 * 1024 * 1024, + }, + }); + registerNativeFullTextTelemetry(writer, workerData.indexLabel, 'writer'); + fullTextTelemetryCollector.beforeClose(writer); + await writer.close({ mode: 'rollback' }); + fullTextTelemetryCollector.afterClose(writer); + analytics.addAnalyticsListener((_metrics, final) => { + if (final) + parentPort.postMessage({ + type: 'job-final-analytics', + collectorStopped: !fullTextTelemetryCollector.collecting, + }); + }); + }) +); diff --git a/unitTests/resources/fixtures/fullTextTelemetryShutdown.cjs b/unitTests/resources/fixtures/fullTextTelemetryShutdown.cjs new file mode 100644 index 0000000000..29ad3a0471 --- /dev/null +++ b/unitTests/resources/fixtures/fullTextTelemetryShutdown.cjs @@ -0,0 +1,77 @@ +'use strict'; + +const { parentPort, workerData } = require('node:worker_threads'); +const { onStartup } = require('#src/utility/lifecycle'); +const { onMessageByType } = require('#src/server/threads/manageThreads'); +const { closeScopeOnShutdown } = require('#src/components/scopeShutdown'); + +function waitForMessage(type) { + return new Promise((resolve) => { + const listener = (message) => { + if (message.type !== type) return; + parentPort.off('message', listener); + resolve(); + }; + parentPort.on('message', listener); + }); +} + +// Run after the configuration startup callbacks, then hold before HTTP starts binding. +onStartup(() => + onStartup(async () => { + const { loadFullTextNativeBinding } = require('#src/resources/indexes/fullTextNativeBinding'); + const { + initializeFullTextTelemetry, + registerNativeFullTextTelemetry, + fullTextTelemetryCollector, + } = require('#src/resources/analytics/fullTextTelemetryRuntime'); + const analytics = require('#src/resources/analytics/write'); + const { waitFor } = require('../../waitFor'); + initializeFullTextTelemetry(); + analytics.setAnalyticsEnabled(true); + await waitFor(() => fullTextTelemetryCollector.collecting, 5000); + const binding = await loadFullTextNativeBinding(); + const options = { + path: workerData.indexPath, + indexId: workerData.indexLabel, + generation: 'shutdown-generation', + fields: [{ name: 'title' }], + analyzer: 'english@2', + positions: true, + limits: { + indexingThreads: 2, + searchThreads: 2, + writerMemoryBytes: 64 * 1024 * 1024, + maxQueuedCommands: 16, + maxQueuedBytes: 64 * 1024 * 1024, + maxBatchBytes: 8 * 1024 * 1024, + }, + }; + const writer = await binding.openNativeFullTextIndex(options); + registerNativeFullTextTelemetry(writer, workerData.indexLabel, 'writer'); + await writer.applyMutationBatch({ upserts: [{ id: 'widget', fields: { title: 'widget' } }], deletes: [] }); + const previous = await binding.openNativeFullTextIndex({ + ...options, + path: workerData.indexPath + '-previous', + indexId: workerData.indexLabel + '-previous', + }); + registerNativeFullTextTelemetry(previous, workerData.indexLabel + '-previous', 'writer'); + fullTextTelemetryCollector.beforeClose(previous); + await previous.close({ mode: 'rollback' }); + fullTextTelemetryCollector.afterClose(previous); + onMessageByType('shutdown', (message) => { + closeScopeOnShutdown(async () => { + fullTextTelemetryCollector.beforeClose(writer); + const released = waitForMessage('release-close'); + parentPort.postMessage({ type: 'native-close-held', collecting: fullTextTelemetryCollector.collecting }); + await released; + await writer.close({ mode: 'rollback' }); + fullTextTelemetryCollector.afterClose(writer); + parentPort.postMessage({ type: 'native-closed', state: writer.telemetry().native.state }); + }, message.exitDeadline !== undefined); + }); + const released = waitForMessage('release-startup'); + parentPort.postMessage({ type: 'native-ready', pending: fullTextTelemetryCollector.hasPendingRows }); + await released; + }) +); diff --git a/unitTests/resources/fixtures/fullTextUpgradeHelpers.cjs b/unitTests/resources/fixtures/fullTextUpgradeHelpers.cjs index 5dd611986e..2b15ad33c0 100644 --- a/unitTests/resources/fixtures/fullTextUpgradeHelpers.cjs +++ b/unitTests/resources/fixtures/fullTextUpgradeHelpers.cjs @@ -2,11 +2,19 @@ const { execFileSync } = require('node:child_process'); const path = require('node:path'); +const { readFileSync } = require('node:fs'); +const assert = require('node:assert'); + +const fixture = readFileSync(path.join(__dirname, 'fullTextUpgradeProcess.cjs'), 'utf8'); function previousFullTextProcess(mode, options, documents, payload, version = '0.5.0') { + const root = process.env.HARPER_FULLTEXT_UPGRADE_ROOT; + assert(root && path.isAbsolute(root), 'HARPER_FULLTEXT_UPGRADE_ROOT must name an absolute provisioned fixture root'); + assert(['0.5.0', '0.7.0', '0.8.0'].includes(version)); try { return JSON.parse( - execFileSync(process.execPath, [path.join(__dirname, 'fullTextUpgradeProcess.cjs')], { + execFileSync(process.execPath, ['--eval', fixture], { + cwd: path.join(root, version), input: JSON.stringify({ mode, options, documents, payload, version }), encoding: 'utf8', timeout: 15_000, diff --git a/unitTests/resources/fixtures/fullTextUpgradeProcess.cjs b/unitTests/resources/fixtures/fullTextUpgradeProcess.cjs index 69a04de413..6f2c7bc279 100644 --- a/unitTests/resources/fixtures/fullTextUpgradeProcess.cjs +++ b/unitTests/resources/fixtures/fullTextUpgradeProcess.cjs @@ -7,13 +7,7 @@ const assert = require('node:assert'); for await (const chunk of process.stdin) chunks.push(chunk); const { mode, options, documents, payload, version = '0.5.0' } = JSON.parse(Buffer.concat(chunks)); assert(['0.5.0', '0.7.0', '0.8.0'].includes(version)); - const binding = await import( - version === '0.8.0' - ? '@harperfast/fulltext-v08/native' - : version === '0.7.0' - ? '@harperfast/fulltext-v07/native' - : '@harperfast/fulltext-previous/native' - ); + const binding = await import('@harperfast/fulltext/native'); const info = await binding.runtimeInfo(); assert.strictEqual(info.packageVersion, version); assert.strictEqual(info.tantivyVersion, version !== '0.5.0' ? '0.27.0' : '0.26.2'); diff --git a/unitTests/resources/fixtures/fulltext-upgrade/0.5.0/package-lock.json b/unitTests/resources/fixtures/fulltext-upgrade/0.5.0/package-lock.json new file mode 100644 index 0000000000..bea62c1769 --- /dev/null +++ b/unitTests/resources/fixtures/fulltext-upgrade/0.5.0/package-lock.json @@ -0,0 +1,100 @@ +{ + "name": "harper-fulltext-upgrade-0-5-0", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "harper-fulltext-upgrade-0-5-0", + "version": "1.0.0", + "dependencies": { + "@harperfast/fulltext": "0.5.0" + } + }, + "node_modules/@harperfast/fulltext": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext/-/fulltext-0.5.0.tgz", + "integrity": "sha512-rH2Q3LJLHPhxi8muzPLg00STbO7RFIqAgLA3tpBzSRdsJVuCV9ZMO2StlSZnUAPMnF5wF9SieuhHT9wi1vINBQ==", + "license": "Apache-2.0", + "engines": { + "node": "^22.18.0 || >=24.0.0" + }, + "optionalDependencies": { + "@harperfast/fulltext-darwin-arm64": "0.5.0", + "@harperfast/fulltext-linux-arm64-gnu": "0.5.0", + "@harperfast/fulltext-linux-x64-gnu": "0.5.0", + "@harperfast/fulltext-win32-x64-msvc": "0.5.0" + } + }, + "node_modules/@harperfast/fulltext-darwin-arm64": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-darwin-arm64/-/fulltext-darwin-arm64-0.5.0.tgz", + "integrity": "sha512-0OtKonopmXaP0/9wvpCuo/sklNxOEiQWnVf7aUsthH0B3sKUYT7HGnGtYkzqqUxdsFNlHPBr3IShyrWx9nAWYQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/@harperfast/fulltext-linux-arm64-gnu": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-arm64-gnu/-/fulltext-linux-arm64-gnu-0.5.0.tgz", + "integrity": "sha512-QfNlYUntkal56jZejQ8cbSj8qvUlYv21V2j9xVctkFv+gmfn892kdU4oCg7dR7Qift8/huKUOyL/eGO0qYbgGw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/@harperfast/fulltext-linux-x64-gnu": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-x64-gnu/-/fulltext-linux-x64-gnu-0.5.0.tgz", + "integrity": "sha512-+g5LMXhtODen+pGgmyTxcCuUAlk2NTeFOvr6duqUBVx542Hd/uDqPC9tUMjL12LJ9ci3LFqGwgW3FkxGedT7KA==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/@harperfast/fulltext-win32-x64-msvc": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-win32-x64-msvc/-/fulltext-win32-x64-msvc-0.5.0.tgz", + "integrity": "sha512-PseSz47BMcIOMQdYUFoAUqAEyeHzceX+QQFun0wVCu/PtERPvtPWzzZxV4ofySzz+c9FkJHTSDu4XbAZhkaqkA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + } + } +} diff --git a/unitTests/resources/fixtures/fulltext-upgrade/0.5.0/package.json b/unitTests/resources/fixtures/fulltext-upgrade/0.5.0/package.json new file mode 100644 index 0000000000..b7200f3a96 --- /dev/null +++ b/unitTests/resources/fixtures/fulltext-upgrade/0.5.0/package.json @@ -0,0 +1,8 @@ +{ + "name": "harper-fulltext-upgrade-0-5-0", + "version": "1.0.0", + "private": true, + "dependencies": { + "@harperfast/fulltext": "0.5.0" + } +} diff --git a/unitTests/resources/fixtures/fulltext-upgrade/0.7.0/package-lock.json b/unitTests/resources/fixtures/fulltext-upgrade/0.7.0/package-lock.json new file mode 100644 index 0000000000..b50475e799 --- /dev/null +++ b/unitTests/resources/fixtures/fulltext-upgrade/0.7.0/package-lock.json @@ -0,0 +1,100 @@ +{ + "name": "harper-fulltext-upgrade-0-7-0", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "harper-fulltext-upgrade-0-7-0", + "version": "1.0.0", + "dependencies": { + "@harperfast/fulltext": "0.7.0" + } + }, + "node_modules/@harperfast/fulltext": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext/-/fulltext-0.7.0.tgz", + "integrity": "sha512-elDKwjn1PYRkH3Fpc4pIC/YxgfAUfHECtisyVNBEiK1XQArcKQUYxVUJ1KKsxrcn5qzf2vlYM71SyONlAm5zQQ==", + "license": "Apache-2.0", + "engines": { + "node": "^22.18.0 || >=24.0.0" + }, + "optionalDependencies": { + "@harperfast/fulltext-darwin-arm64": "0.7.0", + "@harperfast/fulltext-linux-arm64-gnu": "0.7.0", + "@harperfast/fulltext-linux-x64-gnu": "0.7.0", + "@harperfast/fulltext-win32-x64-msvc": "0.7.0" + } + }, + "node_modules/@harperfast/fulltext-darwin-arm64": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-darwin-arm64/-/fulltext-darwin-arm64-0.7.0.tgz", + "integrity": "sha512-oamX0LDMKt5o1PXDk0T4SN92Z1OvlmnBWVknSExr4SDqmUit9MonSkVvGcTRQL/fAVHnI7kwlNqpsDAywxOE3Q==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/@harperfast/fulltext-linux-arm64-gnu": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-arm64-gnu/-/fulltext-linux-arm64-gnu-0.7.0.tgz", + "integrity": "sha512-jH8szxy+9Q84/O/J7cRo6GRHIntO73PyzdD0Gx9uX/6fsPLJ0+Ldd0+JFok2ygd8Mv72pmI0pcy0UI7W28vi5w==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/@harperfast/fulltext-linux-x64-gnu": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-x64-gnu/-/fulltext-linux-x64-gnu-0.7.0.tgz", + "integrity": "sha512-36xlr2jZa5VmAeUIcPMTs1MRRpvJ20P9RfqjSu75NLD/Zr7J5o71E4b8IkX3R0qvaya1P54BjNTrmQc8nd/amA==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/@harperfast/fulltext-win32-x64-msvc": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-win32-x64-msvc/-/fulltext-win32-x64-msvc-0.7.0.tgz", + "integrity": "sha512-J2Wfs0w/kYlNaUJvwifMPkWWmfRG3cZme/Ha7ZGLO9zMzVsefLiEDYOHXNPrmJfKW+TebSEAu6Z0NB7H63ix4A==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + } + } +} diff --git a/unitTests/resources/fixtures/fulltext-upgrade/0.7.0/package.json b/unitTests/resources/fixtures/fulltext-upgrade/0.7.0/package.json new file mode 100644 index 0000000000..acc13c8745 --- /dev/null +++ b/unitTests/resources/fixtures/fulltext-upgrade/0.7.0/package.json @@ -0,0 +1,8 @@ +{ + "name": "harper-fulltext-upgrade-0-7-0", + "version": "1.0.0", + "private": true, + "dependencies": { + "@harperfast/fulltext": "0.7.0" + } +} diff --git a/unitTests/resources/fixtures/fulltext-upgrade/0.8.0/package-lock.json b/unitTests/resources/fixtures/fulltext-upgrade/0.8.0/package-lock.json new file mode 100644 index 0000000000..f3dfab2ebf --- /dev/null +++ b/unitTests/resources/fixtures/fulltext-upgrade/0.8.0/package-lock.json @@ -0,0 +1,100 @@ +{ + "name": "harper-fulltext-upgrade-0-8-0", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "harper-fulltext-upgrade-0-8-0", + "version": "1.0.0", + "dependencies": { + "@harperfast/fulltext": "0.8.0" + } + }, + "node_modules/@harperfast/fulltext": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext/-/fulltext-0.8.0.tgz", + "integrity": "sha512-7T+i2PbdsyrXwb4V2mcE23Pk8DwOl12vgE8YodTe/KUeS7tzJp8I/uQOkD5YiW1ga4WasfLVbZrcTi5pULMjAg==", + "license": "Apache-2.0", + "engines": { + "node": "^22.18.0 || >=24.0.0" + }, + "optionalDependencies": { + "@harperfast/fulltext-darwin-arm64": "0.8.0", + "@harperfast/fulltext-linux-arm64-gnu": "0.8.0", + "@harperfast/fulltext-linux-x64-gnu": "0.8.0", + "@harperfast/fulltext-win32-x64-msvc": "0.8.0" + } + }, + "node_modules/@harperfast/fulltext-darwin-arm64": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-darwin-arm64/-/fulltext-darwin-arm64-0.8.0.tgz", + "integrity": "sha512-Egmt0FsOqgTEeZo6YpxMbywTbaO8aH3gDqHpC+lfmrqXAFiDub8NqQ72xSAI0VUfSMXUL83/NoHGX68brZ+upA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/@harperfast/fulltext-linux-arm64-gnu": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-arm64-gnu/-/fulltext-linux-arm64-gnu-0.8.0.tgz", + "integrity": "sha512-YfLa+bjnZUXI5bY3JPOmlkn3DlKXA3zheryUxD+IKTjUvR5ekNoYbl5WTEzYhqRZGZs2o07ct2+Zb3otiUP3Ww==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/@harperfast/fulltext-linux-x64-gnu": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-x64-gnu/-/fulltext-linux-x64-gnu-0.8.0.tgz", + "integrity": "sha512-xLTpOAyNax0FpKKoMsnIRgcyMD1sdCVhKmBXO5pMPB9vHoAoTBoeyGjKxAv2HGqwcOXT5sfhAGmCUpqPlbDtVA==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/@harperfast/fulltext-win32-x64-msvc": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-win32-x64-msvc/-/fulltext-win32-x64-msvc-0.8.0.tgz", + "integrity": "sha512-GKQ9GR8WuN8T4tqZS+smQanLm97lMiA2uZv6pMJk79zh4xtNrC5wOzARx3/J3l3wclI2HCf+iVKgn1OxOfvR7w==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + } + } +} diff --git a/unitTests/resources/fixtures/fulltext-upgrade/0.8.0/package.json b/unitTests/resources/fixtures/fulltext-upgrade/0.8.0/package.json new file mode 100644 index 0000000000..fe8700d0f1 --- /dev/null +++ b/unitTests/resources/fixtures/fulltext-upgrade/0.8.0/package.json @@ -0,0 +1,8 @@ +{ + "name": "harper-fulltext-upgrade-0-8-0", + "version": "1.0.0", + "private": true, + "dependencies": { + "@harperfast/fulltext": "0.8.0" + } +} diff --git a/unitTests/resources/fixtures/installFullTextUpgradeDependencies.mjs b/unitTests/resources/fixtures/installFullTextUpgradeDependencies.mjs new file mode 100644 index 0000000000..1e0ebf0339 --- /dev/null +++ b/unitTests/resources/fixtures/installFullTextUpgradeDependencies.mjs @@ -0,0 +1,36 @@ +import assert from 'node:assert'; +import { execFileSync } from 'node:child_process'; +import { copyFileSync, mkdirSync } from 'node:fs'; +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const [root] = process.argv.slice(2); +assert(root && isAbsolute(root), 'Pass an absolute scratch root for the historical full-text engines'); +const source = join(dirname(fileURLToPath(import.meta.url)), 'fulltext-upgrade'); +const destination = relative(resolve(source, '../../../..'), root); +assert( + destination === '..' || destination.startsWith(`..${sep}`) || isAbsolute(destination), + 'Install historical engines outside the Harper checkout' +); +assert(process.env.npm_execpath, 'Run through npm run test:fulltext:install-previous -- ROOT'); + +for (const version of ['0.5.0', '0.7.0', '0.8.0']) { + const prefix = join(root, version); + mkdirSync(prefix, { recursive: true }); + for (const file of ['package.json', 'package-lock.json']) + copyFileSync(join(source, version, file), join(prefix, file)); + execFileSync( + process.execPath, + [ + process.env.npm_execpath, + 'ci', + '--ignore-scripts', + '--include=optional', + '--no-audit', + '--no-fund', + '--prefix', + prefix, + ], + { stdio: 'inherit' } + ); +} diff --git a/unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js b/unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js index 6d83ccfeec..388dfe59ae 100644 --- a/unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js +++ b/unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js @@ -734,6 +734,7 @@ describe('published full-text 0.5.0, 0.7.0 and 0.8.0 to current migration', func let rootPath; before(async function () { + if (!process.env.HARPER_FULLTEXT_UPGRADE_ROOT) this.skip(); if (!(await supportsPublishedFullTextBinding())) this.skip(); rootPath = setupTestDBPath(); binding = await loadFullTextNativeBinding(); diff --git a/unitTests/resources/nativeFullTextIndexSchemaUpgrade.test.js b/unitTests/resources/nativeFullTextIndexSchemaUpgrade.test.js index beb35931b1..61712a16e9 100644 --- a/unitTests/resources/nativeFullTextIndexSchemaUpgrade.test.js +++ b/unitTests/resources/nativeFullTextIndexSchemaUpgrade.test.js @@ -39,6 +39,7 @@ describe('schema-declared full-text 0.5.0 to 0.7.0 checkpoint upgrade', function this.timeout(60_000); it('fences public search, rebuilds the old checkpoint and replays writes through the production runtime', async function () { + if (!process.env.HARPER_FULLTEXT_UPGRADE_ROOT) this.skip(); if (process.env.HARPER_STORAGE_ENGINE === 'lmdb' || !(await supportsPublishedFullTextBinding())) this.skip(); setupTestDBPath(); const previousMainIsWorker = getWorkerIndex() === 0; diff --git a/unitTests/resources/nativeFullTextIndexUpgradeCrash.test.js b/unitTests/resources/nativeFullTextIndexUpgradeCrash.test.js index 37dc77a1e9..ba6f967f5e 100644 --- a/unitTests/resources/nativeFullTextIndexUpgradeCrash.test.js +++ b/unitTests/resources/nativeFullTextIndexUpgradeCrash.test.js @@ -13,6 +13,7 @@ describe('published native full-text upgrade process crash recovery', function ( this.timeout(120_000); let directory; before(async function () { + if (!process.env.HARPER_FULLTEXT_UPGRADE_ROOT) this.skip(); if (process.env.HARPER_STORAGE_ENGINE === 'lmdb' || !(await supportsPublishedFullTextBinding())) this.skip(); directory = path.join(setupTestDBPath(), 'native-upgrade-crashes'); }); diff --git a/unitTests/resources/nativeFullTextTelemetry.test.js b/unitTests/resources/nativeFullTextTelemetry.test.js index 52357d77ec..7ec71122d8 100644 --- a/unitTests/resources/nativeFullTextTelemetry.test.js +++ b/unitTests/resources/nativeFullTextTelemetry.test.js @@ -9,8 +9,6 @@ const { loadFullTextNativeBinding } = require('#src/resources/indexes/fullTextNa const { FullTextTelemetryCollector, resolveFullTextTelemetryPolicy, - fullTextNativeCounterDelta, - flattenFullTextTelemetry, fullTextTelemetryIndexLabel, } = require('#src/resources/analytics/fullTextTelemetry'); @@ -335,27 +333,112 @@ describe('Published full-text telemetry collection', function () { const recovered = collector.takeRows().find((value) => value.path === 'catalog.Product0/search'); assert.strictEqual(recovered.complete, true, 'dropping a pending observation does not lose cumulative counters'); }); - it('uses exact uint64 deltas above safe integer range and across wrap', () => { + it('evicts the least recently refreshed pending group', async () => { + collector.stop(); + collector = new FullTextTelemetryCollector({ maxPendingRows: 2 }); + collector.setOperatorPolicy({ enabled: true }); + const writers = []; + for (let i = 0; i < 3; i++) { + const { index } = await writer(); + writers.push(index); + collector.register(index, `catalog.Recency${i}/search`, 'writer'); + } + collector.collect(writers[0]); + collector.collect(writers[1]); + collector.collect(writers[0]); + collector.collect(writers[2]); + const rows = collector.takeRows(); + assert.deepStrictEqual( + rows.map((entry) => entry.path), + ['catalog.Recency0/search', 'catalog.Recency2/search'] + ); + assert.strictEqual(rows.at(-1).droppedRows, '1'); + }); + it('preserves a pending final row before evicting a replaceable live row', async () => { + collector.stop(); + collector = new FullTextTelemetryCollector({ maxPendingRows: 2 }); + collector.setOperatorPolicy({ enabled: true }); + const writers = []; + for (let i = 0; i < 3; i++) { + const { index } = await writer(); + writers.push(index); + collector.register(index, `catalog.Final${i}/search`, 'writer'); + } + collector.beforeClose(writers[0]); + await writers[0].close({ mode: 'rollback' }); + collector.afterClose(writers[0]); + collector.collect(writers[1]); + collector.collect(writers[2]); + const rows = collector.takeRows(); + assert.deepStrictEqual( + rows.map((entry) => entry.path), + ['catalog.Final0/search', 'catalog.Final2/search'] + ); + assert.strictEqual(rows[0].native_state, 'closed'); + assert.strictEqual(rows.at(-1).droppedRows, '1'); + }); + it('samples all open handles at final flush without overflowing the pending-row budget', async () => { + collector.stop(); + collector = new FullTextTelemetryCollector({ maxPendingRows: 2 }); + collector.setOperatorPolicy({ enabled: true }); + for (let i = 0; i < 3; i++) { + const { index } = await writer(); + collector.register(index, `catalog.Open${i}/search`, 'writer'); + await index.search({ text: 'widget' }); + } + const rows = collector.collectFinalRows(); + assert.deepStrictEqual( + rows.map((entry) => entry.path), + ['catalog.Open0/search', 'catalog.Open1/search', 'catalog.Open2/search'] + ); + for (const entry of rows) { + assert.strictEqual(entry.native_operations_search_completed, '1'); + assert.strictEqual(entry.droppedRows, '0'); + } + assert.strictEqual(collector.hasPendingRows, false); + }); + it('accumulates exact uint64 deltas above safe integer range and across wrap', async () => { + const { index } = await writer(); + const snapshot = index.telemetry(); + const handle = { telemetry: () => snapshot, configureTelemetry() {} }; + collector.setOperatorPolicy({ enabled: true }); + collector.register(handle, 'catalog.Counter/search', 'writer'); const previous = (1n << 63n) + 1n; - assert.strictEqual(fullTextNativeCounterDelta(previous + 3n, previous), 3n); - assert.strictEqual(fullTextNativeCounterDelta(2n, (1n << 64n) - 2n), 4n); + snapshot.native.operations.search.completed = previous; + collector.collect(handle); + assert.strictEqual(row('writer').native_operations_search_completed, previous.toString()); + snapshot.native.operations.search.completed += 3n; + collector.collect(handle); + assert.strictEqual(row('writer').native_operations_search_completed, (previous + 3n).toString()); + snapshot.native.operations.search.completed = (1n << 64n) - 2n; + collector.collect(handle); + assert.strictEqual(row('writer').native_operations_search_completed, ((1n << 64n) - 2n).toString()); + snapshot.native.operations.search.completed = 2n; + collector.collect(handle); + assert.strictEqual(row('writer').native_operations_search_completed, ((1n << 64n) + 2n).toString()); }); it('preserves exact snapshot scalars and unavailable native observations without inventing zeros', async () => { const { index } = await writer(); - const snapshot = index.telemetry(); + let snapshot = index.telemetry(); const huge = (1n << 63n) + 17n; snapshot.native.operations.search.completed = huge; snapshot.native.phases.commit.sumNanoseconds = huge; - const encoded = flattenFullTextTelemetry(snapshot); - assert.strictEqual(encoded.counters.get('native_operations_search_completed'), huge); - assert.strictEqual(encoded.counters.get('native_phases_commit_sumNanoseconds'), huge); - assert.strictEqual(encoded.gauges.native_reader_documents, '1'); - assert.strictEqual(encoded.gauges.native_reader_snapshotBytes, null); + const handle = { telemetry: () => snapshot, configureTelemetry() {} }; + collector.setOperatorPolicy({ enabled: true }); + collector.register(handle, 'catalog.Scalars/search', 'writer'); + collector.collect(handle); + const encoded = row('writer'); + assert.strictEqual(encoded.native_operations_search_completed, huge.toString()); + assert.strictEqual(encoded.native_phases_commit_sumNanoseconds, huge.toString()); + assert.strictEqual(encoded.native_reader_documents, '1'); + assert.strictEqual(encoded.native_reader_snapshotBytes, null); const { native: _native, ...facade } = snapshot; - const unavailable = flattenFullTextTelemetry({ ...facade, available: false, reason: 'telemetry unavailable' }); - assert.strictEqual(unavailable.gauges.available, false); - assert.strictEqual(unavailable.gauges.native_reader_documents, undefined); - assert.strictEqual(unavailable.counters.get('logicalBatches_completed'), 1n); + snapshot = { ...facade, available: false, reason: 'telemetry unavailable' }; + collector.collect(handle); + const unavailable = row('writer'); + assert.strictEqual(unavailable.available, false); + assert.strictEqual(unavailable.native_reader_documents, undefined); + assert.strictEqual(unavailable.logicalBatches_completed, '1'); }); it('preserves peer gauge sums, null ages, newest-key selection and closing exclusions', async () => { const { index } = await writer(); diff --git a/unitTests/resources/nativeFullTextTelemetryShutdown.test.js b/unitTests/resources/nativeFullTextTelemetryShutdown.test.js new file mode 100644 index 0000000000..09b2bc4172 --- /dev/null +++ b/unitTests/resources/nativeFullTextTelemetryShutdown.test.js @@ -0,0 +1,216 @@ +'use strict'; + +const assert = require('node:assert'); +const { once } = require('node:events'); +const { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync } = require('node:fs'); +const { join } = require('node:path'); +const YAML = require('yaml'); +const { setupTestDBPath, ensureSystemTables } = require('../testUtils'); +const { startWorker } = require('#src/server/threads/manageThreads'); +const { getFullTextRuntimeConfiguration } = require('#src/resources/indexes/fullTextRuntimeConfiguration'); +const { getConfigFilePath } = require('#src/config/configUtils'); +const { supportsPublishedFullTextBinding } = require('./fullTextNativePlatform'); +const { setRootConfigEntry, readRootConfig } = require('../rootConfigFixture'); +const { waitFor } = require('../waitFor'); +const analytics = require('#src/resources/analytics/write'); +const { + initializeFullTextTelemetry, + fullTextTelemetryCollector, +} = require('#src/resources/analytics/fullTextTelemetryRuntime'); +const { getDatabases } = require('#src/resources/databases'); +const { randomUUID } = require('node:crypto'); +const { jobWorkerOptions } = require('#src/server/jobs/jobRunner'); +const { JOB_STATUS_ENUM, SYSTEM_TABLE_NAMES } = require('#src/utility/hdbTerms'); + +it('waits for an in-flight analytics window before delivering exactly one final window', async function () { + this.timeout(60_000); + await ensureSystemTables(); + const worker = startWorker(require.resolve('./fixtures/analyticsExitFlush.cjs'), { + name: 'http', + autoRestart: false, + }); + const messages = []; + worker.prependListener('message', (message) => messages.push(structuredClone(message))); + try { + const [code] = await once(worker, 'exit'); + assert.strictEqual(code, 0); + assert.deepStrictEqual(messages.find((message) => message.type === 'flush-completed')?.order, [ + 'ordinary-started', + 'exit-requested', + 'ordinary-delivered', + 'final-delivered', + ]); + assert.strictEqual(messages.filter((message) => message.type === 'analytics-report').length, 2); + } finally { + if (worker.threadId !== -1) await worker.terminate(); + } +}); + +it('flushes actual job-worker analytics before cleanup acknowledgement and persists native counters', async function () { + this.timeout(60_000); + if (process.env.HARPER_STORAGE_ENGINE === 'lmdb' || !(await supportsPublishedFullTextBinding())) this.skip(); + await ensureSystemTables(); + const id = randomUUID(); + const label = `job-exit-${id}`; + const indexPath = join(setupTestDBPath(), label); + const jobs = getDatabases().system[SYSTEM_TABLE_NAMES.JOB_TABLE_NAME]; + const savedConfig = readFileSync(getConfigFilePath(), 'utf8'); + const wasEnabled = analytics.checkAnalyticsEnabled(false); + let worker; + try { + initializeFullTextTelemetry(); + analytics.setAnalyticsEnabled(true); + setRootConfigEntry('analytics', { + ...readRootConfig().analytics, + fullText: { enabled: true, intervalMilliseconds: 60_000 }, + }); + await waitFor(() => fullTextTelemetryCollector.collecting, 5000); + await jobs.put({ + id, + type: 'delete_transaction_logs_before', + status: JOB_STATUS_ENUM.IN_PROGRESS, + created_datetime: Date.now(), + start_datetime: Date.now(), + request: { operation: 'delete_transaction_logs_before', database: 'system', timestamp: 0 }, + }); + worker = startWorker(require.resolve('#src/server/jobs/jobProcess'), { + ...jobWorkerOptions(id), + execArgv: [ + ...process.execArgv.filter((argument) => argument.startsWith('--conditions=')), + '--require', + require.resolve('./fixtures/fullTextTelemetryJobExit.cjs'), + ], + workerData: { + addPorts: [], + addThreadIds: [], + indexPath, + indexLabel: label, + fullTextRuntimeConfiguration: getFullTextRuntimeConfiguration(), + }, + }); + const messages = []; + worker.prependListener('message', (message) => messages.push(structuredClone(message))); + const workerId = worker.threadId; + const [code] = await once(worker, 'exit'); + assert.strictEqual(code, 0, JSON.stringify(messages)); + assert.strictEqual((await jobs.get(id)).status, JOB_STATUS_ENUM.COMPLETE); + const finalIndex = messages.findIndex((message) => message.type === 'job-final-analytics'); + const ackIndex = messages.findIndex((message) => message.type === 'job_cleanup_complete'); + assert.ok(finalIndex >= 0, 'real jobProcess must invoke terminal analytics'); + assert.strictEqual(messages[finalIndex].collectorStopped, true); + assert.ok(ackIndex > finalIndex, 'terminal analytics must finish before job cleanup acknowledgement'); + const finalReportIndex = messages.findLastIndex((message) => message.type === 'analytics-report'); + assert.ok(finalReportIndex > finalIndex && finalReportIndex < ackIndex); + const stored = []; + for (const { value } of getDatabases().system.hdb_raw_analytics.primaryStore.getRange({ + start: false, + end: Infinity, + })) + if (value.threadId === workerId) stored.push(...value.metrics); + assert.ok( + stored.some( + (row) => row.path === label && row.native_state === 'closed' && row.native_operations_close_completed === '1' + ) + ); + } finally { + try { + if (worker && worker.threadId !== -1) await worker.terminate(); + await jobs.delete(id); + } finally { + writeFileSync(getConfigFilePath(), savedConfig); + analytics.setAnalyticsEnabled(wasEnabled); + rmSync(indexPath, { recursive: true, force: true, maxRetries: 5 }); + } + } +}); + +it('delivers final native telemetry after deferred scope close and before worker exit', async function () { + this.timeout(60_000); + if (!(await supportsPublishedFullTextBinding())) this.skip(); + await ensureSystemTables(); + const root = mkdtempSync(join(setupTestDBPath(), 'telemetry-shutdown-')); + for (const directory of ['database', 'log', 'components', 'keys']) mkdirSync(join(root, directory)); + const config = YAML.parse(readFileSync(getConfigFilePath(), 'utf8')); + config.rootPath = root; + config.analytics = { ...config.analytics, fullText: { enabled: true, intervalMilliseconds: 60_000 } }; + writeFileSync(join(root, 'harper-config.yaml'), YAML.stringify(config)); + const label = `shutdown-${process.pid}`; + const savedConfig = readFileSync(getConfigFilePath(), 'utf8'); + const wasEnabled = analytics.checkAnalyticsEnabled(false); + let worker; + try { + initializeFullTextTelemetry(); + analytics.setAnalyticsEnabled(true); + setRootConfigEntry('analytics', { + ...readRootConfig().analytics, + fullText: { enabled: true, intervalMilliseconds: 60_000 }, + }); + await waitFor(() => fullTextTelemetryCollector.collecting, 5000); + worker = startWorker(require.resolve('#src/server/threads/threadServer'), { + autoRestart: false, + name: 'http', + execArgv: [ + ...process.execArgv.filter((argument) => argument.startsWith('--conditions=')), + '--require', + require.resolve('./fixtures/fullTextTelemetryShutdown.cjs'), + ], + env: { ...process.env, ROOTPATH: root, HARPER_SAFE_MODE: 'true' }, + workerData: { + addPorts: [], + addThreadIds: [], + indexPath: join(root, 'index'), + indexLabel: label, + fullTextRuntimeConfiguration: getFullTextRuntimeConfiguration(), + }, + }); + const messages = []; + // Preserve the wire report independently of main's storage normalization. + worker.prependListener('message', (message) => messages.push(structuredClone(message))); + const workerId = worker.threadId; + const exited = once(worker, 'exit'); + async function waitForMessage(type, timeout = 15_000) { + while (!messages.some((message) => message.type === type)) + await once(worker, 'message', { signal: AbortSignal.timeout(timeout) }).catch((error) => { + throw new Error(`Missing ${type}; received ${JSON.stringify(messages)}`, { cause: error }); + }); + return messages.find((message) => message.type === type); + } + assert.strictEqual((await waitForMessage('native-ready', 45_000)).pending, true); + worker.postMessage({ type: 'shutdown', restartNumber: 2, exitDeadline: Date.now() + 15_000 }); + worker.postMessage({ type: 'release-startup' }); + const held = await waitForMessage('native-close-held'); + worker.postMessage({ type: 'release-close' }); + const [code] = await exited; + assert.strictEqual(code, 0); + assert.strictEqual(held.collecting, true, 'collector must outlive the deferred native owner'); + assert.strictEqual(messages.find((message) => message.type === 'native-closed')?.state, 'closed'); + const rows = messages.flatMap((message) => message.report?.metrics ?? []).filter((row) => row.path === label); + const final = rows.find((row) => row.native_state === 'closed'); + assert.ok(final, `closed native snapshot must be delivered before worker exit: ${JSON.stringify(messages)}`); + assert.strictEqual(final.native_operations_close_completed, '1'); + const pending = messages + .flatMap((message) => message.report?.metrics ?? []) + .find((row) => row.path === label + '-previous'); + assert.strictEqual(pending?.native_state, 'closed', 'shutdown must preserve an already pending closed-handle row'); + const persisted = () => { + for (const { value } of getDatabases().system.hdb_raw_analytics.primaryStore.getRange({ + start: false, + end: Infinity, + })) { + if (value.threadId !== workerId) continue; + const row = value.metrics.find((row) => row.path === label && row.native_state === 'closed'); + if (row) return row; + } + }; + await waitFor(persisted, 5000); + assert.strictEqual(persisted().native_operations_close_completed, '1'); + } finally { + try { + if (worker && worker.threadId !== -1) await worker.terminate(); + } finally { + writeFileSync(getConfigFilePath(), savedConfig); + analytics.setAnalyticsEnabled(wasEnabled); + rmSync(root, { recursive: true, force: true, maxRetries: 5 }); + } + } +}); From 8d6a83467c281b836acde8ffdf6a29feef7f4a0a Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 09:23:31 -0700 Subject: [PATCH 12/26] fix: provision deployed migration fixtures and retain flush error logging --- .github/workflows/integration-tests.yml | 20 ++ .github/workflows/unit-test.yml | 4 +- .../resources/full-text-search.test.ts | 279 +++++++++--------- resources/FULL_TEXT.md | 4 +- resources/analytics/write.ts | 4 +- .../resources/fixtures/analyticsExitFlush.cjs | 27 +- .../fixtures/fullTextUpgradeProcess.cjs | 4 + .../installFullTextUpgradeDependencies.mjs | 12 +- .../nativeFullTextTelemetryShutdown.test.js | 25 +- 9 files changed, 230 insertions(+), 149 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 4a68c6ed2b..7dd635d784 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -194,6 +194,10 @@ jobs: fi dpkg-query -W openssh-server + - name: Provision the historical full-text integration engine + timeout-minutes: 3 + run: npm run test:fulltext:install-previous -- "${{ runner.temp }}/fulltext-upgrade" 0.5.0 + - name: Run Integration Test Shard ${{ matrix.shard }} timeout-minutes: 15 env: @@ -205,6 +209,7 @@ jobs: # package.json's minor advances. HARPER_PREVIOUS_MINOR_PATH: /tmp/harper-prev-minor-52/node_modules/harper:/tmp/harper-prev-minor-53/node_modules/harper HARPER_INTEGRATION_TEST_INSTALL_SCRIPT: dist/bin/harper.js + HARPER_FULLTEXT_UPGRADE_ROOT: ${{ runner.temp }}/fulltext-upgrade # Linux runners have working inotify, so the file-watch-dependent TLS regression # tests (cert-key-reload) must run rather than skip — this pipeline owns #2382. HARPER_TEST_REQUIRE_FILE_WATCHERS: '1' @@ -288,12 +293,17 @@ jobs: timeout-minutes: 5 run: npm install --ignore-scripts --prefix "${{ runner.temp }}/harperdb-legacy" harperdb@4 + - name: Provision the historical full-text integration engine + timeout-minutes: 3 + run: npm run test:fulltext:install-previous -- "${{ runner.temp }}/fulltext-upgrade" 0.5.0 + - name: Run Integration Test Shard ${{ matrix.shard }} timeout-minutes: 20 env: HARPER_INTEGRATION_TEST_LOG_DIR: ${{ runner.temp }}/harper-integration-test-logs HARPER_LEGACY_VERSION_PATH: ${{ runner.temp }}/harperdb-legacy/node_modules/harperdb HARPER_INTEGRATION_TEST_INSTALL_SCRIPT: dist/bin/harper.js + HARPER_FULLTEXT_UPGRADE_ROOT: ${{ runner.temp }}/fulltext-upgrade run: npm run test:integration:all -- --shard=${{ matrix.shard }}/6 - name: Upload Harper server logs @@ -343,12 +353,17 @@ jobs: timeout-minutes: 5 run: npm install --ignore-scripts + - name: Provision the historical full-text integration engine + timeout-minutes: 3 + run: npm run test:fulltext:install-previous -- "${{ runner.temp }}/fulltext-upgrade" 0.5.0 + - name: Run Integration Test Shard ${{ matrix.shard }} timeout-minutes: 15 env: HARPER_INTEGRATION_TEST_LOG_DIR: /tmp/harper-integration-test-logs HARPER_RUNTIME: bun HARPER_INTEGRATION_TEST_INSTALL_SCRIPT: dist/bin/harper.js + HARPER_FULLTEXT_UPGRADE_ROOT: ${{ runner.temp }}/fulltext-upgrade run: | npm run test:integration:all -- --shard=${{ matrix.shard }}/6 @@ -407,6 +422,10 @@ jobs: # (#914). Only non-secure, non-operations TCP ports are affected; secure/replication/ops paths # keep running on Node, so this exercises the uWS request/streaming/WS/GraphQL/Fastify-fallback # path against the full integration suite. + - name: Provision the historical full-text integration engine + timeout-minutes: 3 + run: npm run test:fulltext:install-previous -- "${{ runner.temp }}/fulltext-upgrade" 0.5.0 + - name: Run Integration Test Shard ${{ matrix.shard }} timeout-minutes: 15 env: @@ -415,6 +434,7 @@ jobs: HARPER_LEGACY_51_PATH: /tmp/harper-legacy-51/node_modules/harper/dist/bin/harper.js HARPER_UWS_HTTP: 1 HARPER_INTEGRATION_TEST_INSTALL_SCRIPT: dist/bin/harper.js + HARPER_FULLTEXT_UPGRADE_ROOT: ${{ runner.temp }}/fulltext-upgrade run: | npm run test:integration:all -- --shard=${{ matrix.shard }}/6 diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 02873dbfd7..203e602474 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -305,8 +305,6 @@ jobs: name: Full-text native plane (${{ matrix.os }}, Node.js v26) runs-on: ${{ matrix.os }} timeout-minutes: 15 - env: - HARPER_FULLTEXT_UPGRADE_ROOT: ${{ runner.temp }}/fulltext-upgrade strategy: fail-fast: false matrix: @@ -335,6 +333,8 @@ jobs: run: node --input-type=module --eval "const { validateFullTextNativeBinding } = await import('./dist/resources/indexes/fullTextNativeBinding.js'); await validateFullTextNativeBinding(await import('@harperfast/fulltext/native'));" - name: Native full-text lifecycle test + env: + HARPER_FULLTEXT_UPGRADE_ROOT: ${{ runner.temp }}/fulltext-upgrade run: >- npx mocha unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js unitTests/resources/nativeFullTextTelemetry.test.js diff --git a/integrationTests/resources/full-text-search.test.ts b/integrationTests/resources/full-text-search.test.ts index f109812603..8eb5d52dd9 100644 --- a/integrationTests/resources/full-text-search.test.ts +++ b/integrationTests/resources/full-text-search.test.ts @@ -650,147 +650,152 @@ suite('deployed full-text fields and native search', (ctx: ContextWithHarper) => await waitForIds(query('asteroid'), []); }); - test('upgrades a real 0.5 checkpoint on two workers and recovers after an old-engine refusal', async () => { - const records = [ - { id: 'upgrade-added', title: 'added waterproof' }, - { id: 'upgrade-ad', title: 'ad' }, - { id: 'upgrade-deleted', title: 'removed migrationwitness' }, - ]; - for (const record of records) { - await request(`/LifecycleProduct/${record.id}`, { method: 'PUT', body: JSON.stringify(record) }, 204); - } - await waitForReady('LifecycleProduct'); - const searchLifecycle = (text: string) => - request('/LifecycleProduct/', { method: 'QUERY', body: JSON.stringify(query(text)) }); - assert.deepStrictEqual(ids(await searchLifecycle('added')), ['upgrade-added']); - assert.deepStrictEqual(ids(await searchLifecycle('ad')), ['upgrade-ad']); - const before = (await request('/FullTextState/')).LifecycleProduct; - const previousLogPath = join(ctx.harper.logDir ?? join(ctx.harper.dataRootDir, 'log'), 'hdb.log'); - const previousLogLength = (await readFile(previousLogPath, 'utf8')).length; - await killHarper(ctx); - assert.ok(ctx.harper.process.exitCode !== null || ctx.harper.process.signalCode !== null); - const digest = (value: string) => createHash('sha256').update(value).digest('hex'); - const configuration = { - path: join(before.storePath, `${digest('LifecycleProduct/catalogSearch')}.fulltext`), - indexId: 'fulltext:LifecycleProduct/catalogSearch', - generation: digest(`${before.tableId}:${before.generations.catalogSearch}`), - fields: [{ name: 'title', weight: 3 }], - analyzer: 'english@2', - stopWords: true, - positions: true, - surfaceTerms: true, - synonyms: [], - limits: { - indexingThreads: 2, - searchThreads: 2, - writerMemoryBytes: 64 * 1024 * 1024, - maxQueuedCommands: 16, - maxQueuedBytes: 64 * 1024 * 1024, - maxBatchBytes: 8 * 1024 * 1024, - }, - }; - const checkpoint = native.inspectNativeFullTextIndex(configuration); - assert.strictEqual(checkpoint.state, 'checkpointed', 'seed must replace the actual declared index'); - const reader = await native.openNativeFullTextReader(configuration); - const documents = []; - try { + test( + 'upgrades a real 0.5 checkpoint on two workers and recovers after an old-engine refusal', + { skip: !process.env.HARPER_FULLTEXT_UPGRADE_ROOT }, + async () => { + previousFullTextProcess('probe'); + const records = [ + { id: 'upgrade-added', title: 'added waterproof' }, + { id: 'upgrade-ad', title: 'ad' }, + { id: 'upgrade-deleted', title: 'removed migrationwitness' }, + ]; for (const record of records) { - const hits = (await reader.search({ text: record.title, mode: 'all', exactTotal: true })).hits; - assert.strictEqual(hits.length, 1, record.id); - documents.push({ id: hits[0].id, version: hits[0].version, fields: { title: record.title } }); + await request(`/LifecycleProduct/${record.id}`, { method: 'PUT', body: JSON.stringify(record) }, 204); } - } finally { - await reader.close(); - } - await native.resetNativeFullTextIndex({ path: configuration.path, indexId: configuration.indexId }); - const seeded = previousFullTextProcess( - 'seed', - configuration, - [...documents, { id: 'native-only-ghost', version: '1', fields: { title: 'migrationghost' } }], - checkpoint.committedPayload - ); - assert.deepStrictEqual(seeded.inspection, checkpoint); - assert.deepStrictEqual(seeded.ghostIds, ['native-only-ghost']); - assert.deepStrictEqual(native.inspectNativeFullTextIndex(configuration), { - state: 'incompatible', - code: 'E_IDENTITY_MISMATCH', - }); - await startHarper(ctx, OPTIONS); - const deadline = Date.now() + 30_000; - let ready = false; - do { - let response: Response; - let body: string; + await waitForReady('LifecycleProduct'); + const searchLifecycle = (text: string) => + request('/LifecycleProduct/', { method: 'QUERY', body: JSON.stringify(query(text)) }); + assert.deepStrictEqual(ids(await searchLifecycle('added')), ['upgrade-added']); + assert.deepStrictEqual(ids(await searchLifecycle('ad')), ['upgrade-ad']); + const before = (await request('/FullTextState/')).LifecycleProduct; + const previousLogPath = join(ctx.harper.logDir ?? join(ctx.harper.dataRootDir, 'log'), 'hdb.log'); + const previousLogLength = (await readFile(previousLogPath, 'utf8')).length; + await killHarper(ctx); + assert.ok(ctx.harper.process.exitCode !== null || ctx.harper.process.signalCode !== null); + const digest = (value: string) => createHash('sha256').update(value).digest('hex'); + const configuration = { + path: join(before.storePath, `${digest('LifecycleProduct/catalogSearch')}.fulltext`), + indexId: 'fulltext:LifecycleProduct/catalogSearch', + generation: digest(`${before.tableId}:${before.generations.catalogSearch}`), + fields: [{ name: 'title', weight: 3 }], + analyzer: 'english@2', + stopWords: true, + positions: true, + surfaceTerms: true, + synonyms: [], + limits: { + indexingThreads: 2, + searchThreads: 2, + writerMemoryBytes: 64 * 1024 * 1024, + maxQueuedCommands: 16, + maxQueuedBytes: 64 * 1024 * 1024, + maxBatchBytes: 8 * 1024 * 1024, + }, + }; + const checkpoint = native.inspectNativeFullTextIndex(configuration); + assert.strictEqual(checkpoint.state, 'checkpointed', 'seed must replace the actual declared index'); + const reader = await native.openNativeFullTextReader(configuration); + const documents = []; try { - response = await fetch(`${ctx.harper.httpURL}/LifecycleProduct/?catalogSearch=matches=ad`, { - headers: { Authorization: authorization }, - signal: AbortSignal.timeout(Math.max(1, deadline - Date.now())), - }); - body = await response.text(); - } catch (error) { - if (error.name === 'TimeoutError') - assert.fail('upgraded public search did not become ready before the deadline'); - assert.ok(!response || response.status === 200, String(error)); - assert.ok( - error.name === 'TypeError' && ['UND_ERR_SOCKET', 'ECONNRESET'].includes(error.cause?.code), - String(error.cause ?? error) - ); - await delay(25); - continue; + for (const record of records) { + const hits = (await reader.search({ text: record.title, mode: 'all', exactTotal: true })).hits; + assert.strictEqual(hits.length, 1, record.id); + documents.push({ id: hits[0].id, version: hits[0].version, fields: { title: record.title } }); + } + } finally { + await reader.close(); } - assert.ok(response.status === 200 || response.status === 503, body); - if (response.status === 200) { - const records = JSON.parse(body); - assert.ok(Array.isArray(records)); - assert.deepStrictEqual(ids(records), ['upgrade-ad'], 'no request may expose the old stemming'); - ready = true; - } else assert.ok(body.length > 0); - if (!ready) await delay(25); - } while (!ready && Date.now() < deadline); - assert.ok(ready, 'upgraded public search did not become ready before the deadline'); - await waitForReady('LifecycleProduct'); - const mismatch = /fulltext:LifecycleProduct\/catalogSearch.*is incompatible \(E_IDENTITY_MISMATCH\); rebuilding/; - const logPath = join(ctx.harper.logDir ?? join(ctx.harper.dataRootDir, 'log'), 'hdb.log'); - const logDeadline = Date.now() + 10_000; - let upgradeLog: string; - do { - upgradeLog = (await readFile(previousLogPath, 'utf8')).slice(previousLogLength); - if (logPath !== previousLogPath) upgradeLog += await readFile(logPath, 'utf8'); - if (mismatch.test(upgradeLog)) break; - await delay(25); - } while (Date.now() < logDeadline); - assert.match(upgradeLog, mismatch); - assert.deepStrictEqual(ids(await searchLifecycle('added')), ['upgrade-added']); - assert.deepStrictEqual(await searchLifecycle('migrationghost'), []); - await request('/LifecycleProduct/upgrade-deleted', { method: 'DELETE' }); - assert.deepStrictEqual(await searchLifecycle('migrationwitness'), []); - await request( - '/LifecycleProduct/upgrade-added', - { - method: 'PATCH', - body: JSON.stringify({ title: 'added waterproof recovered' }), - }, - 204 - ); - assert.deepStrictEqual(ids(await searchLifecycle('recovered')), ['upgrade-added']); - const after = (await request('/FullTextState/')).LifecycleProduct; - assert.strictEqual(after.tableId, before.tableId); - assert.deepStrictEqual(after.generations, before.generations); - await killHarper(ctx); - const refused = previousFullTextProcess('inspect', configuration); - assert.deepStrictEqual(refused.inspection, { state: 'incompatible', code: 'E_IDENTITY_MISMATCH' }); - assert.strictEqual(refused.openError, 'E_IDENTITY_MISMATCH'); - assert.strictEqual(previousFullTextProcess('reset', configuration).resetError, 'E_INDEX_CORRUPT'); - assert.strictEqual(native.inspectNativeFullTextIndex(configuration).state, 'checkpointed'); - await startHarper(ctx, OPTIONS); - await waitForReady('LifecycleProduct'); - assert.deepStrictEqual(ids(await searchLifecycle('recovered')), ['upgrade-added']); - assert.deepStrictEqual(ids(await searchLifecycle('ad')), ['upgrade-ad']); - assert.deepStrictEqual(await searchLifecycle('migrationwitness'), []); - assert.deepStrictEqual(await searchLifecycle('migrationghost'), []); - await waitForReady('Product'); - await waitForIds(query('waterproof'), ['one', 'two']); - }); + await native.resetNativeFullTextIndex({ path: configuration.path, indexId: configuration.indexId }); + const seeded = previousFullTextProcess( + 'seed', + configuration, + [...documents, { id: 'native-only-ghost', version: '1', fields: { title: 'migrationghost' } }], + checkpoint.committedPayload + ); + assert.deepStrictEqual(seeded.inspection, checkpoint); + assert.deepStrictEqual(seeded.ghostIds, ['native-only-ghost']); + assert.deepStrictEqual(native.inspectNativeFullTextIndex(configuration), { + state: 'incompatible', + code: 'E_IDENTITY_MISMATCH', + }); + await startHarper(ctx, OPTIONS); + const deadline = Date.now() + 30_000; + let ready = false; + do { + let response: Response; + let body: string; + try { + response = await fetch(`${ctx.harper.httpURL}/LifecycleProduct/?catalogSearch=matches=ad`, { + headers: { Authorization: authorization }, + signal: AbortSignal.timeout(Math.max(1, deadline - Date.now())), + }); + body = await response.text(); + } catch (error) { + if (error.name === 'TimeoutError') + assert.fail('upgraded public search did not become ready before the deadline'); + assert.ok(!response || response.status === 200, String(error)); + assert.ok( + error.name === 'TypeError' && ['UND_ERR_SOCKET', 'ECONNRESET'].includes(error.cause?.code), + String(error.cause ?? error) + ); + await delay(25); + continue; + } + assert.ok(response.status === 200 || response.status === 503, body); + if (response.status === 200) { + const records = JSON.parse(body); + assert.ok(Array.isArray(records)); + assert.deepStrictEqual(ids(records), ['upgrade-ad'], 'no request may expose the old stemming'); + ready = true; + } else assert.ok(body.length > 0); + if (!ready) await delay(25); + } while (!ready && Date.now() < deadline); + assert.ok(ready, 'upgraded public search did not become ready before the deadline'); + await waitForReady('LifecycleProduct'); + const mismatch = /fulltext:LifecycleProduct\/catalogSearch.*is incompatible \(E_IDENTITY_MISMATCH\); rebuilding/; + const logPath = join(ctx.harper.logDir ?? join(ctx.harper.dataRootDir, 'log'), 'hdb.log'); + const logDeadline = Date.now() + 10_000; + let upgradeLog: string; + do { + upgradeLog = (await readFile(previousLogPath, 'utf8')).slice(previousLogLength); + if (logPath !== previousLogPath) upgradeLog += await readFile(logPath, 'utf8'); + if (mismatch.test(upgradeLog)) break; + await delay(25); + } while (Date.now() < logDeadline); + assert.match(upgradeLog, mismatch); + assert.deepStrictEqual(ids(await searchLifecycle('added')), ['upgrade-added']); + assert.deepStrictEqual(await searchLifecycle('migrationghost'), []); + await request('/LifecycleProduct/upgrade-deleted', { method: 'DELETE' }); + assert.deepStrictEqual(await searchLifecycle('migrationwitness'), []); + await request( + '/LifecycleProduct/upgrade-added', + { + method: 'PATCH', + body: JSON.stringify({ title: 'added waterproof recovered' }), + }, + 204 + ); + assert.deepStrictEqual(ids(await searchLifecycle('recovered')), ['upgrade-added']); + const after = (await request('/FullTextState/')).LifecycleProduct; + assert.strictEqual(after.tableId, before.tableId); + assert.deepStrictEqual(after.generations, before.generations); + await killHarper(ctx); + const refused = previousFullTextProcess('inspect', configuration); + assert.deepStrictEqual(refused.inspection, { state: 'incompatible', code: 'E_IDENTITY_MISMATCH' }); + assert.strictEqual(refused.openError, 'E_IDENTITY_MISMATCH'); + assert.strictEqual(previousFullTextProcess('reset', configuration).resetError, 'E_INDEX_CORRUPT'); + assert.strictEqual(native.inspectNativeFullTextIndex(configuration).state, 'checkpointed'); + await startHarper(ctx, OPTIONS); + await waitForReady('LifecycleProduct'); + assert.deepStrictEqual(ids(await searchLifecycle('recovered')), ['upgrade-added']); + assert.deepStrictEqual(ids(await searchLifecycle('ad')), ['upgrade-ad']); + assert.deepStrictEqual(await searchLifecycle('migrationwitness'), []); + assert.deepStrictEqual(await searchLifecycle('migrationghost'), []); + await waitForReady('Product'); + await waitForIds(query('waterproof'), ['one', 'two']); + } + ); test('preserves index generation and physical paths and protects declarations after restart', async () => { await request( diff --git a/resources/FULL_TEXT.md b/resources/FULL_TEXT.md index af622a8486..5e2873640f 100644 --- a/resources/FULL_TEXT.md +++ b/resources/FULL_TEXT.md @@ -92,7 +92,7 @@ Harper pins `@harperfast/fulltext` to **0.9.0**, embedding **Tantivy 0.27.0**, a ### Upgrading from fulltext 0.7.0 -Harper's existing indexes written by fulltext 0.7.0 contain text fields only and remain reusable with 0.9.0. Genuine 0.8.0 checkpoints are also reused without rebuilding; the regression advances them with 0.9.0 and reopens them with 0.8.0. This binary-level check is not a supported older-Harper operator downgrade procedure. This upgrade does not add native filter metadata or change their storage identity. Query limits and fuzzy controls do not require rebuilding. Stop all processes sharing a local index before switching package versions; never mix facade/native binaries or engine versions against a live directory. New installations omit `storage.fullText.query`; defaults apply automatically. An explicitly configured query block must be removed before returning to Harper 5.4.0-alpha.1 or alpha.2, whose validators reject it. Existing configuration is not rewritten. Returning to fulltext 0.7.0 after 0.8.0 has written to an index is not a qualified operator rollback procedure. +On nodes that already ran v5.4 prereleases with fulltext 0.7.0, existing indexes contain text fields only and remain reusable with 0.9.0. Compatible checkpoints from v5.4 prereleases with fulltext 0.8.0 are also reused without rebuilding; the regression advances them with 0.9.0 and reopens them with 0.8.0. This binary-level check is not a supported older-Harper operator downgrade procedure. This upgrade does not add native filter metadata or change their storage identity. Query limits and fuzzy controls do not require rebuilding. Stop all processes sharing a local index before switching package versions; never mix facade/native binaries or engine versions against a live directory. New installations omit `storage.fullText.query`; defaults apply automatically. An explicitly configured query block must be removed before returning to Harper 5.4.0-alpha.1 or alpha.2, whose validators reject it. Existing configuration is not rewritten. Returning to fulltext 0.7.0 after 0.8.0 has written to an index is not a qualified operator rollback procedure. ### Upgrading an existing native index @@ -207,7 +207,7 @@ analytics: The implied periodic interval is 60,000 milliseconds per handle. Calls are staggered across existing writer/readers, and their rows use Harper's existing analytics flush, storage, retention and protected `get_analytics` interface. Successful handle closes also collect final counters, so the interval is not a limit on total snapshot or export frequency. Collection never opens a cold index. To tune it, set `intervalMilliseconds` between 1,000 and 2,147,483,647; synchronous snapshots and export work have measurable cost, so qualify the intended number of indexes, reader churn and cadence. No production overhead bound is claimed. -Set `enabled: false` or remove the block to stop scheduled polling and persistent full-text export. A synchronous observation already executing completes. Editing unrelated settings does not restart collection. Invalid edits or a missing configuration file retain the last valid telemetry policy and in-process overrides. Recreating an unchanged file preserves them. To disable collection, write a valid file with `enabled: false` or remove the telemetry block; valid edits retire overrides only when their matching effective value changes after environment overlays. Disabling analytics also disables its full-text collection demand. Internal registered consumers can independently request observations; removing the last demand stops the timer. These are implementation hooks, not a public SDK, and currently have no production caller. Consumer-only observations do not enable persistent analytics. With operator export enabled, persisted fields reflect the union of operator and consumer recording requests. +Set `enabled: false` or remove the block to stop scheduled polling and persistent full-text export, unless environment overlays or in-process overrides still enable it. A synchronous observation already executing completes. Editing unrelated settings does not restart collection. Invalid edits or a missing configuration file retain the last valid telemetry policy and in-process overrides. Recreating an unchanged file preserves them. To disable collection, write a valid file with `enabled: false` or remove the telemetry block and remove any environment or in-process setting that keeps it enabled; valid edits retire overrides only when their matching effective value changes after environment overlays. Disabling analytics also disables its full-text collection demand. Internal registered consumers can independently request observations; removing the last demand stops the timer. These are implementation hooks, not a public SDK, and currently have no production caller. Consumer-only observations do not enable persistent analytics. With operator export enabled, persisted fields reflect the union of operator and consumer recording requests. Workers and the receiving main thread enforce their current export policy independently. During configuration transitions, the receiver can discard worker rows, including a final close observation; these policy-filtered rows are not counted in `droppedRows` and may have no later replacement. diff --git a/resources/analytics/write.ts b/resources/analytics/write.ts index 1eb84494e4..38a0e5e857 100644 --- a/resources/analytics/write.ts +++ b/resources/analytics/write.ts @@ -234,7 +234,9 @@ function sendAnalytics() { // One more timer turn: a stall that ran past the deadline is recorded by the event loop // sampler's overdue firing, which libuv runs before a timer armed now, and the read below // would otherwise reset that sample away. - sendAnalyticsTimeout = setTimeout(flushAnalytics, 0).unref(); + sendAnalyticsTimeout = setTimeout(() => { + void flushAnalytics().catch((error) => harperLogger.warn('Analytics flush failed', error)); + }, 0).unref(); }, analyticsDelay).unref(); } diff --git a/unitTests/resources/fixtures/analyticsExitFlush.cjs b/unitTests/resources/fixtures/analyticsExitFlush.cjs index 9bd45c9278..d9fcccce01 100644 --- a/unitTests/resources/fixtures/analyticsExitFlush.cjs +++ b/unitTests/resources/fixtures/analyticsExitFlush.cjs @@ -1,11 +1,36 @@ 'use strict'; const assert = require('node:assert'); -const { parentPort } = require('node:worker_threads'); +const { parentPort, workerData } = require('node:worker_threads'); const env = require('#src/utility/environment/environmentManager'); env.initSync(); const analytics = require('#src/resources/analytics/write'); parentPort.ref(); +if (workerData.failPeriodic) { + const { pinLogConfig } = require('../../logConfigFixture'); + const { readFileSync } = require('node:fs'); + const { getLogFilePath } = require('#src/utility/logging/harper_logger'); + const { waitFor } = require('../../waitFor'); + const restore = pinLogConfig(); + analytics.setAnalyticsEnabled(true); + analytics.recordAction(() => { + throw new Error('periodic-metric-regression'); + }, 'failing-periodic-metric'); + waitFor(() => { + try { + const log = readFileSync(getLogFilePath(), 'utf8'); + return log.includes('Analytics flush failed') && log.includes('periodic-metric-regression'); + } catch { + return false; + } + }, 10_000).then(async () => { + await analytics.flushAnalyticsForExit(); + restore(); + parentPort.postMessage({ type: 'periodic-error-logged' }); + parentPort.close(); + }); + return; +} const order = []; analytics.setAnalyticsEnabled(true); analytics.addAnalyticsListener((_metrics, final) => order.push(final ? 'final-delivered' : 'ordinary-delivered')); diff --git a/unitTests/resources/fixtures/fullTextUpgradeProcess.cjs b/unitTests/resources/fixtures/fullTextUpgradeProcess.cjs index 6f2c7bc279..2bb24cae75 100644 --- a/unitTests/resources/fixtures/fullTextUpgradeProcess.cjs +++ b/unitTests/resources/fixtures/fullTextUpgradeProcess.cjs @@ -11,6 +11,10 @@ const assert = require('node:assert'); const info = await binding.runtimeInfo(); assert.strictEqual(info.packageVersion, version); assert.strictEqual(info.tantivyVersion, version !== '0.5.0' ? '0.27.0' : '0.26.2'); + if (mode === 'probe') { + console.log(JSON.stringify({ runtimeInfo: info })); + return; + } let index; let ghostIds; try { diff --git a/unitTests/resources/fixtures/installFullTextUpgradeDependencies.mjs b/unitTests/resources/fixtures/installFullTextUpgradeDependencies.mjs index 1e0ebf0339..1a87aeb79f 100644 --- a/unitTests/resources/fixtures/installFullTextUpgradeDependencies.mjs +++ b/unitTests/resources/fixtures/installFullTextUpgradeDependencies.mjs @@ -4,7 +4,13 @@ import { copyFileSync, mkdirSync } from 'node:fs'; import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; -const [root] = process.argv.slice(2); +const [root, ...requestedVersions] = process.argv.slice(2); +const supportedVersions = ['0.5.0', '0.7.0', '0.8.0']; +const versions = requestedVersions.length ? requestedVersions : supportedVersions; +assert( + versions.every((version) => supportedVersions.includes(version)), + 'Unsupported historical full-text version' +); assert(root && isAbsolute(root), 'Pass an absolute scratch root for the historical full-text engines'); const source = join(dirname(fileURLToPath(import.meta.url)), 'fulltext-upgrade'); const destination = relative(resolve(source, '../../../..'), root); @@ -14,7 +20,7 @@ assert( ); assert(process.env.npm_execpath, 'Run through npm run test:fulltext:install-previous -- ROOT'); -for (const version of ['0.5.0', '0.7.0', '0.8.0']) { +for (const version of versions) { const prefix = join(root, version); mkdirSync(prefix, { recursive: true }); for (const file of ['package.json', 'package-lock.json']) @@ -31,6 +37,6 @@ for (const version of ['0.5.0', '0.7.0', '0.8.0']) { '--prefix', prefix, ], - { stdio: 'inherit' } + { stdio: 'inherit', timeout: 120_000 } ); } diff --git a/unitTests/resources/nativeFullTextTelemetryShutdown.test.js b/unitTests/resources/nativeFullTextTelemetryShutdown.test.js index 09b2bc4172..41e9ce4694 100644 --- a/unitTests/resources/nativeFullTextTelemetryShutdown.test.js +++ b/unitTests/resources/nativeFullTextTelemetryShutdown.test.js @@ -22,6 +22,25 @@ const { randomUUID } = require('node:crypto'); const { jobWorkerOptions } = require('#src/server/jobs/jobRunner'); const { JOB_STATUS_ENUM, SYSTEM_TABLE_NAMES } = require('#src/utility/hdbTerms'); +it('logs a failing periodic analytics window without preventing the final flush', async function () { + this.timeout(60_000); + await ensureSystemTables(); + const worker = startWorker(require.resolve('./fixtures/analyticsExitFlush.cjs'), { + name: 'http', + autoRestart: false, + workerData: { failPeriodic: true }, + }); + const messages = []; + worker.prependListener('message', (message) => messages.push(structuredClone(message))); + try { + const [code] = await once(worker, 'exit', { signal: AbortSignal.timeout(55_000) }); + assert.strictEqual(code, 0); + assert.ok(messages.some((message) => message.type === 'periodic-error-logged')); + } finally { + if (worker.threadId !== -1) await worker.terminate(); + } +}); + it('waits for an in-flight analytics window before delivering exactly one final window', async function () { this.timeout(60_000); await ensureSystemTables(); @@ -32,7 +51,7 @@ it('waits for an in-flight analytics window before delivering exactly one final const messages = []; worker.prependListener('message', (message) => messages.push(structuredClone(message))); try { - const [code] = await once(worker, 'exit'); + const [code] = await once(worker, 'exit', { signal: AbortSignal.timeout(55_000) }); assert.strictEqual(code, 0); assert.deepStrictEqual(messages.find((message) => message.type === 'flush-completed')?.order, [ 'ordinary-started', @@ -91,7 +110,7 @@ it('flushes actual job-worker analytics before cleanup acknowledgement and persi const messages = []; worker.prependListener('message', (message) => messages.push(structuredClone(message))); const workerId = worker.threadId; - const [code] = await once(worker, 'exit'); + const [code] = await once(worker, 'exit', { signal: AbortSignal.timeout(55_000) }); assert.strictEqual(code, 0, JSON.stringify(messages)); assert.strictEqual((await jobs.get(id)).status, JOB_STATUS_ENUM.COMPLETE); const finalIndex = messages.findIndex((message) => message.type === 'job-final-analytics'); @@ -167,7 +186,7 @@ it('delivers final native telemetry after deferred scope close and before worker // Preserve the wire report independently of main's storage normalization. worker.prependListener('message', (message) => messages.push(structuredClone(message))); const workerId = worker.threadId; - const exited = once(worker, 'exit'); + const exited = once(worker, 'exit', { signal: AbortSignal.timeout(55_000) }); async function waitForMessage(type, timeout = 15_000) { while (!messages.some((message) => message.type === type)) await once(worker, 'message', { signal: AbortSignal.timeout(timeout) }).catch((error) => { From 66214c38fb5dcc797d0e1ab3a45ed22aaafb63fb Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 09:30:39 -0700 Subject: [PATCH 13/26] fix: isolate periodic and terminal analytics failures --- dependencies.md | 6 ++--- .../analytics/fullTextTelemetryRuntime.ts | 2 ++ resources/analytics/write.ts | 17 +++++++------ .../resources/fixtures/analyticsExitFlush.cjs | 12 ++++++++-- .../fixtures/fullTextUpgradeHelpers.cjs | 6 ++++- .../installFullTextUpgradeDependencies.mjs | 4 ++-- .../nativeFullTextTelemetryShutdown.test.js | 24 +++++++++++++++++++ 7 files changed, 54 insertions(+), 17 deletions(-) diff --git a/dependencies.md b/dependencies.md index 32a5a3c532..a1e66def15 100644 --- a/dependencies.md +++ b/dependencies.md @@ -302,6 +302,6 @@ This is the inverse of the entries below — a dependency we take deliberate ste ## Historical full-text upgrade engines (isolated test fixtures) - Need for usage: Released fulltext 0.5.0, 0.7.0 and 0.8.0 write genuine old checkpoints; the current engine must rebuild incompatible indexes or reuse compatible ones, and prior engines verify safe reopen or downgrade refusal. Source-record IDs and checkpoints are generated during the tests, so static index files do not replace these checks. -- Installation boundary: Harper's normal dependency tree contains only the current optional engine. Separate exact-pinned manifests and integrity-checked lockfiles live under `unitTests/resources/fixtures/fulltext-upgrade/`; the dedicated native CI job provisions their packages outside the checkout. The previous-engine child exits before the current engine accesses the same index. Production code never imports historical engines or relaxes current ABI validation. -- Local qualification: Run `npm run test:fulltext:install-previous -- /absolute/scratch/fulltext-upgrade`, then set `HARPER_FULLTEXT_UPGRADE_ROOT` to that directory when running the native lifecycle, schema-upgrade and crash-recovery suites. Ordinary resource tests skip historical migration cases when the variable is absent; an explicit but invalid installation fails. CI supplies it and forbids pending tests. -- Cost and removal: The dedicated job installs three root packages and their supported platform prebuilds. Other development and CI installs incur no historical-package cost. Renovate leaves fixture versions fixed; remove each manifest, lockfile and matching coverage when migration from that version is no longer supported. +- Installation boundary: Harper's normal dependency tree contains only the current optional engine. Separate exact-pinned manifests and integrity-checked lockfiles live under `unitTests/resources/fixtures/fulltext-upgrade/`; the native CI job provisions all three outside the checkout, and existing integration lanes provision only 0.5.0. The previous-engine child exits before the current engine accesses the same index. Production code never imports historical engines or relaxes current ABI validation. +- Local qualification: Run `npm run test:fulltext:install-previous -- /absolute/scratch/fulltext-upgrade`, then set `HARPER_FULLTEXT_UPGRADE_ROOT` to that directory when running the native lifecycle, schema-upgrade, crash-recovery or deployed full-text integration suites. Ordinary resource tests skip historical migration cases when the variable is absent; an explicit but invalid installation fails. CI supplies it; the native unit job also forbids pending tests. +- Cost and removal: The dedicated job installs three root packages and their supported platform prebuilds. Integration lanes install only the 0.5.0 fixture; ordinary development installs incur no historical-package cost. Renovate leaves fixture versions fixed; remove each manifest, lockfile and matching coverage when migration from that version is no longer supported. diff --git a/resources/analytics/fullTextTelemetryRuntime.ts b/resources/analytics/fullTextTelemetryRuntime.ts index 2947c3a5f6..b01297f0bd 100644 --- a/resources/analytics/fullTextTelemetryRuntime.ts +++ b/resources/analytics/fullTextTelemetryRuntime.ts @@ -59,6 +59,8 @@ export function initializeFullTextTelemetry(): void { const rows = fullTextTelemetryCollector.collectFinalRows(); if (persistenceActive && checkAnalyticsEnabled(false)) metrics.push(...rows); } + } catch { + reportFailure(); } finally { removeListener?.(); fullTextTelemetryCollector.stop(); diff --git a/resources/analytics/write.ts b/resources/analytics/write.ts index 38a0e5e857..475d2c3dcc 100644 --- a/resources/analytics/write.ts +++ b/resources/analytics/write.ts @@ -206,12 +206,9 @@ export function flushAnalyticsForExit(): Promise { clearTimeout(sendAnalyticsTimeout); sendAnalyticsTimeout = null; return (exitAnalyticsFlush = contextStorage.exit(async () => { - try { - await runningAnalyticsFlush; - } finally { - if (checkAnalyticsEnabled(false)) await flushAnalytics(true); - else for (const listener of [...analyticsListeners]) listener([], true); - } + await runningAnalyticsFlush?.catch(() => {}); + if (checkAnalyticsEnabled(false)) await flushAnalytics(true); + else for (const listener of [...analyticsListeners]) listener([], true); })); } @@ -234,12 +231,14 @@ function sendAnalytics() { // One more timer turn: a stall that ran past the deadline is recorded by the event loop // sampler's overdue firing, which libuv runs before a timer armed now, and the read below // would otherwise reset that sample away. - sendAnalyticsTimeout = setTimeout(() => { - void flushAnalytics().catch((error) => harperLogger.warn('Analytics flush failed', error)); - }, 0).unref(); + sendAnalyticsTimeout = setTimeout(flushPeriodicAnalytics, 0).unref(); }, analyticsDelay).unref(); } +function flushPeriodicAnalytics(): void { + void flushAnalytics().catch((error) => log.error?.('Analytics flush failed', error)); +} + function flushAnalytics(final = false): Promise { const run = () => collectAnalytics(final); const result = runningAnalyticsFlush ? runningAnalyticsFlush.then(run, run) : run(); diff --git a/unitTests/resources/fixtures/analyticsExitFlush.cjs b/unitTests/resources/fixtures/analyticsExitFlush.cjs index d9fcccce01..cc93b6e298 100644 --- a/unitTests/resources/fixtures/analyticsExitFlush.cjs +++ b/unitTests/resources/fixtures/analyticsExitFlush.cjs @@ -33,7 +33,10 @@ if (workerData.failPeriodic) { } const order = []; analytics.setAnalyticsEnabled(true); -analytics.addAnalyticsListener((_metrics, final) => order.push(final ? 'final-delivered' : 'ordinary-delivered')); +analytics.addAnalyticsListener((_metrics, final) => { + order.push(final ? 'final-delivered' : 'ordinary-delivered'); + if (final && workerData.failFinal) throw new Error('final-window-regression'); +}); analytics.recordAction(() => { order.push('ordinary-started'); queueMicrotask(() => { @@ -42,13 +45,18 @@ analytics.recordAction(() => { assert.strictEqual(analytics.flushAnalyticsForExit(), completed); completed.then( () => { + assert.ok(!workerData.failFinal); parentPort.postMessage({ type: 'flush-completed', order }); parentPort.close(); }, (error) => { - throw error; + assert.ok(workerData.failFinal); + assert.match(error.message, /final-window-regression/); + parentPort.postMessage({ type: 'final-error-propagated', order }); + parentPort.close(); } ); }); + if (workerData.failOverlap) throw new Error('overlapping-periodic-regression'); return { count: 1 }; }, 'shutdown-overlap'); diff --git a/unitTests/resources/fixtures/fullTextUpgradeHelpers.cjs b/unitTests/resources/fixtures/fullTextUpgradeHelpers.cjs index 2b15ad33c0..55a099bc50 100644 --- a/unitTests/resources/fixtures/fullTextUpgradeHelpers.cjs +++ b/unitTests/resources/fixtures/fullTextUpgradeHelpers.cjs @@ -2,7 +2,7 @@ const { execFileSync } = require('node:child_process'); const path = require('node:path'); -const { readFileSync } = require('node:fs'); +const { readFileSync, existsSync } = require('node:fs'); const assert = require('node:assert'); const fixture = readFileSync(path.join(__dirname, 'fullTextUpgradeProcess.cjs'), 'utf8'); @@ -11,6 +11,10 @@ function previousFullTextProcess(mode, options, documents, payload, version = '0 const root = process.env.HARPER_FULLTEXT_UPGRADE_ROOT; assert(root && path.isAbsolute(root), 'HARPER_FULLTEXT_UPGRADE_ROOT must name an absolute provisioned fixture root'); assert(['0.5.0', '0.7.0', '0.8.0'].includes(version)); + assert( + existsSync(path.join(root, version, 'node_modules', '@harperfast', 'fulltext', 'package.json')), + `Historical full-text ${version} is not provisioned under ${root}` + ); try { return JSON.parse( execFileSync(process.execPath, ['--eval', fixture], { diff --git a/unitTests/resources/fixtures/installFullTextUpgradeDependencies.mjs b/unitTests/resources/fixtures/installFullTextUpgradeDependencies.mjs index 1a87aeb79f..c93f981052 100644 --- a/unitTests/resources/fixtures/installFullTextUpgradeDependencies.mjs +++ b/unitTests/resources/fixtures/installFullTextUpgradeDependencies.mjs @@ -8,8 +8,8 @@ const [root, ...requestedVersions] = process.argv.slice(2); const supportedVersions = ['0.5.0', '0.7.0', '0.8.0']; const versions = requestedVersions.length ? requestedVersions : supportedVersions; assert( - versions.every((version) => supportedVersions.includes(version)), - 'Unsupported historical full-text version' + versions.every((version) => supportedVersions.includes(version)) && new Set(versions).size === versions.length, + 'Unsupported or duplicate historical full-text version' ); assert(root && isAbsolute(root), 'Pass an absolute scratch root for the historical full-text engines'); const source = join(dirname(fileURLToPath(import.meta.url)), 'fulltext-upgrade'); diff --git a/unitTests/resources/nativeFullTextTelemetryShutdown.test.js b/unitTests/resources/nativeFullTextTelemetryShutdown.test.js index 41e9ce4694..abf7f50088 100644 --- a/unitTests/resources/nativeFullTextTelemetryShutdown.test.js +++ b/unitTests/resources/nativeFullTextTelemetryShutdown.test.js @@ -41,6 +41,30 @@ it('logs a failing periodic analytics window without preventing the final flush' } }); +for (const mode of ['failOverlap', 'failFinal']) + it(`preserves final-flush rejection ownership (${mode})`, async function () { + this.timeout(60_000); + await ensureSystemTables(); + const worker = startWorker(require.resolve('./fixtures/analyticsExitFlush.cjs'), { + name: 'http', + autoRestart: false, + workerData: { [mode]: true }, + }); + const messages = []; + worker.prependListener('message', (message) => messages.push(structuredClone(message))); + try { + const [code] = await once(worker, 'exit', { signal: AbortSignal.timeout(55_000) }); + assert.strictEqual(code, 0); + assert.ok( + messages.some( + (message) => message.type === (mode === 'failFinal' ? 'final-error-propagated' : 'flush-completed') + ) + ); + } finally { + if (worker.threadId !== -1) await worker.terminate(); + } + }); + it('waits for an in-flight analytics window before delivering exactly one final window', async function () { this.timeout(60_000); await ensureSystemTables(); From 9ebacd944d50029328b6ff54340cde60208e9155 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 09:54:13 -0700 Subject: [PATCH 14/26] test: fail closed on missing CI upgrade fixtures and observe config reads --- .github/workflows/integration-tests.yml | 8 ++++---- .../resources/full-text-search.test.ts | 2 +- .../fullTextTelemetryConfiguration.test.js | 20 ++++++++++++------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 7dd635d784..d4b8c79ff3 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -418,14 +418,14 @@ jobs: mkdir -p /tmp/harper-legacy-51 npm install --ignore-scripts --prefix /tmp/harper-legacy-51 harper@5.1.22 - # Serve the plaintext app HTTP port(s) through uWebSockets.js instead of the Node http server - # (#914). Only non-secure, non-operations TCP ports are affected; secure/replication/ops paths - # keep running on Node, so this exercises the uWS request/streaming/WS/GraphQL/Fastify-fallback - # path against the full integration suite. - name: Provision the historical full-text integration engine timeout-minutes: 3 run: npm run test:fulltext:install-previous -- "${{ runner.temp }}/fulltext-upgrade" 0.5.0 + # Serve the plaintext app HTTP port(s) through uWebSockets.js instead of the Node http server + # (#914). Only non-secure, non-operations TCP ports are affected; secure/replication/ops paths + # keep running on Node, so this exercises the uWS request/streaming/WS/GraphQL/Fastify-fallback + # path against the full integration suite. - name: Run Integration Test Shard ${{ matrix.shard }} timeout-minutes: 15 env: diff --git a/integrationTests/resources/full-text-search.test.ts b/integrationTests/resources/full-text-search.test.ts index 8eb5d52dd9..07869720be 100644 --- a/integrationTests/resources/full-text-search.test.ts +++ b/integrationTests/resources/full-text-search.test.ts @@ -652,7 +652,7 @@ suite('deployed full-text fields and native search', (ctx: ContextWithHarper) => test( 'upgrades a real 0.5 checkpoint on two workers and recovers after an old-engine refusal', - { skip: !process.env.HARPER_FULLTEXT_UPGRADE_ROOT }, + { skip: !process.env.HARPER_FULLTEXT_UPGRADE_ROOT && !process.env.CI }, async () => { previousFullTextProcess('probe'); const records = [ diff --git a/unitTests/config/fullTextTelemetryConfiguration.test.js b/unitTests/config/fullTextTelemetryConfiguration.test.js index f630b44e36..8f076a301c 100644 --- a/unitTests/config/fullTextTelemetryConfiguration.test.js +++ b/unitTests/config/fullTextTelemetryConfiguration.test.js @@ -2,7 +2,6 @@ const assert = require('node:assert'); const { mkdtempSync, writeFileSync, rmSync } = require('node:fs'); const { join } = require('node:path'); const { stringify } = require('yaml'); -const { setTimeout: delay } = require('node:timers/promises'); const { RootConfigWatcher } = require('#src/config/RootConfigWatcher'); const { watchFullTextTelemetryConfiguration } = require('#src/config/fullTextTelemetryConfiguration'); const { resolveFullTextTelemetryPolicy } = require('#src/resources/analytics/fullTextTelemetry'); @@ -52,6 +51,16 @@ describe('Live full-text telemetry configuration', function () { watcher = new RootConfigWatcher(file); stop = watchFullTextTelemetryConfiguration((policy) => samples.push(policy), watcher); } + async function removeConfigAfterObservedRead() { + await watcher.ready; + const reads = watcher._readCountForTests; + rmSync(file); + watcher.handleChange(); + assert.ok( + watcher._readCountForTests > reads, + 'the watcher must finish a real ENOENT read before checking retention' + ); + } it('initializes from the ready nested file policy without an eager aggregate/default application', async () => { writeFileSync(file, stringify(root({ enabled: true, histograms: true }))); watcher = new RootConfigWatcher(file); @@ -108,8 +117,7 @@ describe('Live full-text telemetry configuration', function () { observe({ enabled: true }, samples); await waitFor(() => samples.at(-1)?.enabled === true); const count = samples.length; - rmSync(file); - await delay(250); + await removeConfigAfterObservedRead(); assert.strictEqual(samples.at(-1).enabled, true); assert.strictEqual(samples.length, count); assert.strictEqual(env.getConfigOverrides()[PARAM + '_histograms'], true); @@ -125,8 +133,7 @@ describe('Live full-text telemetry configuration', function () { const samples = []; observe({ enabled: true }, samples); await waitFor(() => samples.at(-1)?.enabled === true); - rmSync(file); - await delay(250); + await removeConfigAfterObservedRead(); assert.strictEqual(samples.at(-1).enabled, true); const retainedCount = samples.length; writeFileSync(file, stringify(root({ enabled: 'invalid' }))); @@ -146,8 +153,7 @@ describe('Live full-text telemetry configuration', function () { writeFileSync(file, stringify(root({ enabled: false, intervalMilliseconds: 3000 }))); await waitFor(() => watcher.config?.analytics.fullText.intervalMilliseconds === 3000); assert.strictEqual(samples.at(-1).intervalMilliseconds, 2000); - rmSync(file); - await delay(250); + await removeConfigAfterObservedRead(); assert.strictEqual(samples.at(-1).enabled, true); assert.strictEqual(samples.at(-1).intervalMilliseconds, 2000); }); From 204dc59383aa6ae2162ffb14c9f65426285eb64d Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 10:31:59 -0700 Subject: [PATCH 15/26] fix: settle final analytics writes and refine telemetry metadata --- benchmarks/fulltext-analytics.mjs | 19 +++++++++-- config/DESIGN.md | 4 ++- config/fullTextTelemetryConfiguration.ts | 2 +- dependencies.md | 4 +-- resources/FULL_TEXT.md | 2 +- resources/analytics/DESIGN.md | 4 ++- resources/analytics/read.ts | 2 +- resources/analytics/write.ts | 6 ++-- .../analytics/aggregationCycle.test.js | 6 ++++ .../resources/fixtures/analyticsMainExit.cjs | 34 +++++++++++++++++++ .../resources/nativeFullTextTelemetry.test.js | 25 ++++++++++++++ .../nativeFullTextTelemetryShutdown.test.js | 23 +++++++++++++ 12 files changed, 119 insertions(+), 12 deletions(-) create mode 100644 unitTests/resources/fixtures/analyticsMainExit.cjs diff --git a/benchmarks/fulltext-analytics.mjs b/benchmarks/fulltext-analytics.mjs index fb0b1fc00e..9715dcfdf7 100644 --- a/benchmarks/fulltext-analytics.mjs +++ b/benchmarks/fulltext-analytics.mjs @@ -37,10 +37,23 @@ const git = (...args) => execFileSync('git', args, { cwd: repository, encoding: const source = { head: git('rev-parse', 'HEAD'), dirty: Boolean(git('status', '--porcelain')), - trackedDiffSha256: createHash('sha256') - .update(git('diff', 'HEAD', '--', '.')) - .digest('hex'), + trackedDiffSha256: await trackedDiffFingerprint(), }; +async function trackedDiffFingerprint() { + const hash = createHash('sha256'); + const child = spawn('git', ['diff', 'HEAD', '--', '.'], { cwd: repository, stdio: ['ignore', 'pipe', 'inherit'] }); + const completed = new Promise((resolve, reject) => { + child.once('error', reject); + child.once('close', (code, signal) => { + if (code === 0) resolve(); + else reject(new Error(`Git fingerprint failed: ${signal ?? code}`)); + }); + }); + child.stdout.pipe(hash); + await completed; + return hash.digest('hex'); +} + const fingerprintFiles = [ 'benchmarks/fulltext-analytics.mjs', 'benchmarks/fulltext-analytics.bench.js', diff --git a/config/DESIGN.md b/config/DESIGN.md index 8604f7235c..6a425ea936 100644 --- a/config/DESIGN.md +++ b/config/DESIGN.md @@ -240,7 +240,9 @@ off on the very boot that could not read its configuration. ## `RootConfigWatcher` schedules a trailing verify-read (`config/RootConfigWatcher.ts`) -Chokidar throttles change events for each path for 50ms without a trailing emit, so closely spaced atomic renames can go unseen. After a successful read, `RootConfigWatcher` rearms a 100ms verify-read (`#read(true)`); close, terminal failure or `ConfigReadRetry` clears it. Unchanged raw text skips parsing; parsed changes use the immutable `#lastParsedConfig` so consumer mutations cannot trigger duplicate events (`rootConfigWatcher.test.js`). Root-config `OptionsWatcher`s lack this verification. Telemetry retains the last valid policy and overrides while the root file is absent or invalid. Only valid file edits retire changed overrides or disable collection (`fullTextTelemetryConfiguration.test.js`); unchanged recreation needs no availability event. +Chokidar throttles `EV.CHANGE` events per path for 50ms (`node_modules/chokidar/index.js`) with no trailing emission, so the second of two `atomicWriteFile()` renames landing within 50ms produces no watcher event. After a successful `handleChange` read, `RootConfigWatcher` arms one 100ms (`TRAILING_VERIFY_READ_DELAY_MS`, double the 50ms throttle window) trailing verify-read (`#read(true)`), re-armed on each event and cleared on `close()`, terminal read failure, or when `ConfigReadRetry` takes over. Post-`ready` reads short-circuit when raw file text matches `#lastData` and otherwise compare the parsed object against an unmutated clone (`#lastParsedConfig`) with `isDeepStrictEqual`, so neither the trailing read nor an identical rewrite emits a duplicate `change` even when consumers (`harper_logger`) mutate `this.#config` in place (`unitTests/config/rootConfigWatcher.test.js`). Root-config `OptionsWatcher`s do not schedule a trailing read and remain exposed to the 50ms throttle window. + +Telemetry retains the last valid policy and overrides while the root file is absent or invalid. Only valid file edits retire changed overrides or disable collection (`unitTests/config/fullTextTelemetryConfiguration.test.js`); unchanged recreation needs no availability event. ## `RootConfigWatcher` polls under Bun (`config/RootConfigWatcher.ts`) diff --git a/config/fullTextTelemetryConfiguration.ts b/config/fullTextTelemetryConfiguration.ts index fa30250991..cb618c7172 100644 --- a/config/fullTextTelemetryConfiguration.ts +++ b/config/fullTextTelemetryConfiguration.ts @@ -24,7 +24,7 @@ export function watchFullTextTelemetryConfiguration( } function report() { try { - logger.warn('Could not apply full-text telemetry configuration; retaining the previous valid policy'); + logger.warn?.('Could not apply full-text telemetry configuration; retaining the previous valid policy'); } catch {} } function changed() { diff --git a/dependencies.md b/dependencies.md index a1e66def15..255a6836e8 100644 --- a/dependencies.md +++ b/dependencies.md @@ -302,6 +302,6 @@ This is the inverse of the entries below — a dependency we take deliberate ste ## Historical full-text upgrade engines (isolated test fixtures) - Need for usage: Released fulltext 0.5.0, 0.7.0 and 0.8.0 write genuine old checkpoints; the current engine must rebuild incompatible indexes or reuse compatible ones, and prior engines verify safe reopen or downgrade refusal. Source-record IDs and checkpoints are generated during the tests, so static index files do not replace these checks. -- Installation boundary: Harper's normal dependency tree contains only the current optional engine. Separate exact-pinned manifests and integrity-checked lockfiles live under `unitTests/resources/fixtures/fulltext-upgrade/`; the native CI job provisions all three outside the checkout, and existing integration lanes provision only 0.5.0. The previous-engine child exits before the current engine accesses the same index. Production code never imports historical engines or relaxes current ABI validation. +- Installation boundary: Harper directly pins the current optional engine; all non-development fulltext entries in the lockfile select 0.9.0. Historical migration aliases are removed from the root dependencies. Separate exact-pinned manifests and integrity-checked lockfiles live under `unitTests/resources/fixtures/fulltext-upgrade/`; the native CI job provisions all three outside the checkout, and existing integration lanes provision only 0.5.0. The previous-engine child exits before the current engine accesses the same index. Production code never imports historical engines or relaxes current ABI validation. - Local qualification: Run `npm run test:fulltext:install-previous -- /absolute/scratch/fulltext-upgrade`, then set `HARPER_FULLTEXT_UPGRADE_ROOT` to that directory when running the native lifecycle, schema-upgrade, crash-recovery or deployed full-text integration suites. Ordinary resource tests skip historical migration cases when the variable is absent; an explicit but invalid installation fails. CI supplies it; the native unit job also forbids pending tests. -- Cost and removal: The dedicated job installs three root packages and their supported platform prebuilds. Integration lanes install only the 0.5.0 fixture; ordinary development installs incur no historical-package cost. Renovate leaves fixture versions fixed; remove each manifest, lockfile and matching coverage when migration from that version is no longer supported. +- Cost and removal: The dedicated job installs three root packages and their supported platform prebuilds. Integration lanes install only the 0.5.0 fixture; ordinary installs do not provision these migration fixtures. The pre-existing `@harperfast/integration-testing` development peer still brings Harper 5.3.1 and its fulltext 0.5.0 package into the development lock graph. That separate framework dependency cleanup is tracked in [Implement full-text generation lifecycle, rebuild, eviction, backup, and status](https://github.com/HarperFast/harper/issues/2510); do not force its older Harper runtime onto the current native ABI. Renovate leaves fixture versions fixed; remove each manifest, lockfile and matching coverage when migration from that version is no longer supported. diff --git a/resources/FULL_TEXT.md b/resources/FULL_TEXT.md index 5e2873640f..bfe72152c3 100644 --- a/resources/FULL_TEXT.md +++ b/resources/FULL_TEXT.md @@ -90,7 +90,7 @@ Weight and highlighting changes preserve the physical index and refresh query re Harper pins `@harperfast/fulltext` to **0.9.0**, embedding **Tantivy 0.27.0**, and validates the native runtime capabilities before activation. Full-text activation requires RocksDB and an audited table. Declare new schema tables with `@table(audit: true)` and keep audit logging enabled while an index is declared. LMDB is rejected. A missing or incompatible native package prevents activation. -### Upgrading from fulltext 0.7.0 +### Upgrading from fulltext 0.7.0 or 0.8.0 On nodes that already ran v5.4 prereleases with fulltext 0.7.0, existing indexes contain text fields only and remain reusable with 0.9.0. Compatible checkpoints from v5.4 prereleases with fulltext 0.8.0 are also reused without rebuilding; the regression advances them with 0.9.0 and reopens them with 0.8.0. This binary-level check is not a supported older-Harper operator downgrade procedure. This upgrade does not add native filter metadata or change their storage identity. Query limits and fuzzy controls do not require rebuilding. Stop all processes sharing a local index before switching package versions; never mix facade/native binaries or engine versions against a live directory. New installations omit `storage.fullText.query`; defaults apply automatically. An explicitly configured query block must be removed before returning to Harper 5.4.0-alpha.1 or alpha.2, whose validators reject it. Existing configuration is not rewritten. Returning to fulltext 0.7.0 after 0.8.0 has written to an index is not a qualified operator rollback procedure. diff --git a/resources/analytics/DESIGN.md b/resources/analytics/DESIGN.md index 26ab274a5f..3ec8b338dd 100644 --- a/resources/analytics/DESIGN.md +++ b/resources/analytics/DESIGN.md @@ -19,7 +19,7 @@ it stops at `firstForPeriod + toPeriod` so one cycle's main-thread msgpackr deco a backlog longer than one window and every report written while the cycle ran, because nothing reads below the cursor again. - **A cycle that reads nothing leaves it where it is.** An empty scan is not proof that nothing is - there: `recordAnalytics` does not await its `primaryStore.put`, so a report can be uncommitted and + there: worker report delivery does not await its `primaryStore.put`, so a report can be uncommitted and invisible to the scan while carrying a key older than anything the cycle could compare against. No wall-clock reading is safe to put in the cursor for the same reason. @@ -78,3 +78,5 @@ always a caller's measure. `fullTextTelemetryRuntime.ts` joins removable internal consumer demand with live operator policy. The consumer hook is an internal diagnostic/test seam with no production caller yet. Only operator demand enables persistence; persisted optional fields reflect the union of recording requests. No demand means no timer; no cold handles open. Exact counters are decimal strings; `cumulative` retains the newest worker row instead of averaging. Lost final counts force a new series on the next live observation. Overflow prefers replaceable live rows, ordered by last refresh, before final rows; dropped live totals remain recoverable. Main also filters worker rows by its receive-time policy; transition losses, including final rows, do not increment `droppedRows`. Failed writer closes resume collection while the engine remains installed. `aggregationCycle.test.js` verifies counters above 2^53 survive storage and `get_analytics`. `watchFullTextTelemetryConfiguration` initializes and reloads through the shared watcher's nested root, environment overlay and transactional `refreshConfigSubtree`; it never reads the missing camel-case flat aggregate. Invalid edits retain valid policy. Missing files retain valid policy and overrides, matching the root watcher; valid edits retire only changed overrides. `fullTextTelemetryConfiguration.test.js` pins initialization, override retirement and deletion/recreation. Native runtime/query policy remains frozen separately. Binding telemetry types are structural: `buildDist.test.mjs` checks compilation and emitted declarations without the optional package. + +Final main-thread analytics flush awaits its own raw write and the raw store’s existing commit barrier for worker reports. This is commit completion, not a new disk-sync guarantee; the process shutdown deadline still bounds it. `describe_metric` omits null-valued attributes from inferred metadata rather than describing unavailable scalar observations as objects; stored rows and `get_analytics` retain their null values. diff --git a/resources/analytics/read.ts b/resources/analytics/read.ts index 0bc107ea33..f14271a7c5 100644 --- a/resources/analytics/read.ts +++ b/resources/analytics/read.ts @@ -356,7 +356,7 @@ export async function describeMetric(metric: string): Promise { await runningAnalyticsFlush?.catch(() => {}); if (checkAnalyticsEnabled(false)) await flushAnalytics(true); else for (const listener of [...analyticsListeners]) listener([], true); + if (!parentPort) await RawAnalyticsTable?.primaryStore.committed; })); } @@ -359,7 +360,7 @@ async function collectAnalytics(final: boolean) { type: ANALYTICS_REPORT_TYPE, report, }); - else recordAnalytics({ report }); + else await recordAnalytics({ report }); } export async function recordHostname() { @@ -1440,9 +1441,10 @@ function recordAnalytics(message, worker?) { lastUtilizations.set(worker, worker.performance.eventLoopUtilization()); } report.id = getNextMonotonicTime(); - getRawAnalyticsTable().primaryStore.put(report.id, report); + const written = getRawAnalyticsTable().primaryStore.put(report.id, report); if (!scheduledTasksRunning) startScheduledTasks(); if (LOG_ANALYTICS) lastAppend = logAnalytics(report); + return written; } let lastAppend; let analyticsLog; diff --git a/unitTests/resources/analytics/aggregationCycle.test.js b/unitTests/resources/analytics/aggregationCycle.test.js index 173662fff9..2ac2937a58 100644 --- a/unitTests/resources/analytics/aggregationCycle.test.js +++ b/unitTests/resources/analytics/aggregationCycle.test.js @@ -219,7 +219,13 @@ describe('analytics aggregation cycle', () => { assert.strictEqual(rows[1].completed, '7'); assert.strictEqual(rows[0].mean, undefined); assert.strictEqual(rows[0].ratio, undefined); + assert.strictEqual(rows[1].error, null); const reader = require('#src/resources/analytics/read'); + const description = await reader.describeMetric(metric); + assert.ok( + description.attributes.some((attribute) => attribute.name === 'completed' && attribute.type === 'string') + ); + assert.ok(!description.attributes.some((attribute) => attribute.name === 'error' && attribute.type === 'object')); await waitFor(async () => { for await (const value of await reader.get(metric, { startTime: first, endTime: Date.now() + 1000 })) if (value.completed === '9007199254740997') return true; diff --git a/unitTests/resources/fixtures/analyticsMainExit.cjs b/unitTests/resources/fixtures/analyticsMainExit.cjs new file mode 100644 index 0000000000..ec6ae6a295 --- /dev/null +++ b/unitTests/resources/fixtures/analyticsMainExit.cjs @@ -0,0 +1,34 @@ +'use strict'; + +const assert = require('node:assert'); +const env = require('#src/utility/environment/environmentManager'); +env.initSync(); +const { table } = require('#src/resources/databases'); +const analytics = require('#src/resources/analytics/write'); +const { runStartup } = require('#src/utility/lifecycle'); + +runStartup() + .then(async () => { + if (process.argv[2] === 'write') { + analytics.setAnalyticsEnabled(true); + analytics.recordAction(true, 'final-commit-probe'); + await analytics.flushAnalyticsForExit(); + } else { + const raw = table({ + table: 'hdb_raw_analytics', + database: 'system', + audit: false, + trackDeletes: false, + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'action' }, { name: 'metrics' }], + }); + let found = false; + for (const { value } of raw.primaryStore.getRange({ start: false, end: Infinity })) + if (value.metrics.some((metric) => metric.metric === 'final-commit-probe' && metric.count === 1)) found = true; + assert.ok(found, 'final analytics report must survive exit after flush completion'); + } + process.exit(0); + }) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/unitTests/resources/nativeFullTextTelemetry.test.js b/unitTests/resources/nativeFullTextTelemetry.test.js index 7ec71122d8..622a2feefc 100644 --- a/unitTests/resources/nativeFullTextTelemetry.test.js +++ b/unitTests/resources/nativeFullTextTelemetry.test.js @@ -147,6 +147,31 @@ describe('Published full-text telemetry collection', function () { (histogramCount + 1n).toString() ); }); + it('preserves released cumulative histogram fields across real observations', async () => { + const { options } = await writer(); + const handle = await reader(options); + collector.register(handle, 'catalog.Histograms/search', 'reader'); + collector.setOperatorPolicy({ enabled: true, histograms: true }); + for (let observation = 1; observation <= 2; observation++) { + await handle.search({ text: 'widget' }); + collector.collect(handle); + const sample = row(); + const native = handle.telemetry().native; + const latency = native.operations.search.execution; + assert.strictEqual(sample.native_operations_search_completed, String(observation)); + assert.strictEqual(sample.native_operations_search_dispatched, '0'); + assert.strictEqual(sample.native_operations_search_execution_count, latency.count.toString()); + assert.strictEqual(sample.native_operations_search_execution_sumNanoseconds, latency.sumNanoseconds.toString()); + for (let bucket = 0; bucket < latency.histogram.buckets.length; bucket++) + assert.strictEqual( + sample[`native_operations_search_execution_histogram_buckets_${bucket}`], + latency.histogram.buckets[bucket].toString() + ); + for (const [name, value] of Object.entries(sample)) + if (name.startsWith('native_operations_') && typeof value === 'string' && /^\d+$/.test(value)) + assert.ok(BigInt(value) < 1n << 63n, name); + } + }); it('unions removable consumer options without persisting consumer-only observations', async () => { const { index } = await writer(); collector.register(index, 'catalog.Product/search', 'writer'); diff --git a/unitTests/resources/nativeFullTextTelemetryShutdown.test.js b/unitTests/resources/nativeFullTextTelemetryShutdown.test.js index abf7f50088..c86418fa03 100644 --- a/unitTests/resources/nativeFullTextTelemetryShutdown.test.js +++ b/unitTests/resources/nativeFullTextTelemetryShutdown.test.js @@ -22,6 +22,29 @@ const { randomUUID } = require('node:crypto'); const { jobWorkerOptions } = require('#src/server/jobs/jobRunner'); const { JOB_STATUS_ENUM, SYSTEM_TABLE_NAMES } = require('#src/utility/hdbTerms'); +it('commits main-thread final analytics before exit and reopens the real store', async function () { + this.timeout(60_000); + const { execFile } = require('node:child_process'); + const { promisify } = require('node:util'); + const root = mkdtempSync(join(setupTestDBPath(), 'analytics-main-exit-')); + for (const directory of ['database', 'log', 'components', 'keys']) mkdirSync(join(root, directory)); + const config = YAML.parse(readFileSync(getConfigFilePath(), 'utf8')); + config.rootPath = root; + config.storage.path = join(root, 'database'); + writeFileSync(join(root, 'harper-config.yaml'), YAML.stringify(config)); + const childEnv = { ...process.env, ROOTPATH: root }; + for (const key of ['HARPER_CONFIG', 'HARPER_DEFAULT_CONFIG', 'HARPER_SET_CONFIG']) delete childEnv[key]; + try { + for (const mode of ['write', 'read']) + await promisify(execFile)(process.execPath, [require.resolve('./fixtures/analyticsMainExit.cjs'), mode], { + env: childEnv, + timeout: 25_000, + }); + } finally { + rmSync(root, { recursive: true, force: true, maxRetries: 5 }); + } +}); + it('logs a failing periodic analytics window without preventing the final flush', async function () { this.timeout(60_000); await ensureSystemTables(); From 5c568c38ba5f151e5fb69cb5b639b522b3f9a12b Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 10:56:29 -0700 Subject: [PATCH 16/26] docs: keep config design guidance within merged budget --- config/DESIGN.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/DESIGN.md b/config/DESIGN.md index 6a425ea936..d1a8e749f1 100644 --- a/config/DESIGN.md +++ b/config/DESIGN.md @@ -240,9 +240,9 @@ off on the very boot that could not read its configuration. ## `RootConfigWatcher` schedules a trailing verify-read (`config/RootConfigWatcher.ts`) -Chokidar throttles `EV.CHANGE` events per path for 50ms (`node_modules/chokidar/index.js`) with no trailing emission, so the second of two `atomicWriteFile()` renames landing within 50ms produces no watcher event. After a successful `handleChange` read, `RootConfigWatcher` arms one 100ms (`TRAILING_VERIFY_READ_DELAY_MS`, double the 50ms throttle window) trailing verify-read (`#read(true)`), re-armed on each event and cleared on `close()`, terminal read failure, or when `ConfigReadRetry` takes over. Post-`ready` reads short-circuit when raw file text matches `#lastData` and otherwise compare the parsed object against an unmutated clone (`#lastParsedConfig`) with `isDeepStrictEqual`, so neither the trailing read nor an identical rewrite emits a duplicate `change` even when consumers (`harper_logger`) mutate `this.#config` in place (`unitTests/config/rootConfigWatcher.test.js`). Root-config `OptionsWatcher`s do not schedule a trailing read and remain exposed to the 50ms throttle window. +Chokidar throttles per-path `EV.CHANGE` for 50ms (`node_modules/chokidar/index.js`) without a trailing event, so adjacent `atomicWriteFile()` renames can be missed. A successful `handleChange` arms a 100ms verify-read (`TRAILING_VERIFY_READ_DELAY_MS`, `#read(true)`), re-armed on each event; close, terminal read failure or `ConfigReadRetry` clears it. After `ready`, matching `#lastData` skips parsing; otherwise `isDeepStrictEqual` compares an unmutated clone (`#lastParsedConfig`), preventing duplicate changes after `harper_logger` mutates `#config` in place (`unitTests/config/rootConfigWatcher.test.js`). Root-config `OptionsWatcher`s lack this verification and remain exposed. -Telemetry retains the last valid policy and overrides while the root file is absent or invalid. Only valid file edits retire changed overrides or disable collection (`unitTests/config/fullTextTelemetryConfiguration.test.js`); unchanged recreation needs no availability event. +For `fullTextTelemetryConfiguration.ts` policy retention and override behavior, see [analytics design](../resources/analytics/DESIGN.md#full-text-collection-follows-actual-handle-ownership). ## `RootConfigWatcher` polls under Bun (`config/RootConfigWatcher.ts`) From b46476b6726b81fd43075edc819cf8c79d4c72ab Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 11:56:21 -0700 Subject: [PATCH 17/26] Use the local Harper build for the integration framework peer --- .github/DESIGN.md | 2 + .github/workflows/integration-tests.yml | 2 + .github/workflows/lockfile-npm-compat.yml | 2 +- build-tools/DESIGN.md | 2 + build-tools/bundleDependencies.ts | 5 + dependencies.md | 8 +- integrationTests/README.md | 4 + package-lock.json | 1158 ++--------------- package.json | 4 + .../build-tools/bundleDependencies.test.mjs | 38 +- 10 files changed, 176 insertions(+), 1049 deletions(-) diff --git a/.github/DESIGN.md b/.github/DESIGN.md index 1ac595806f..35d86b6124 100644 --- a/.github/DESIGN.md +++ b/.github/DESIGN.md @@ -30,6 +30,8 @@ Branch protection requires only the `Unit Test` job, which needs every other job Node 26 is the recommended runtime, so every single-version job runs it, and pull requests run unit tests on 24 and 26 (the `lmdb` shard on 26 alone until harper#3349's segfault is fixed) and integration tests on 26 alone. Pushes to `main` run both on 24 and 26. Node 22 runs nightly and on a manual `all` dispatch, as do the native type-strip legs on `22.18.0` and `24.0.0`, the `engines` floors in `package.json`, which are the oldest runtimes that must run the `.ts` sources directly. The nightly schedule runs only on `main`, so a push to a release branch runs that full set: a patch release still ships to Node 22 users. The cherry-pick job dispatches a release branch's unit tests on Node 26 (or 24 when that branch's workflow predates 26) and its integration tests on the Node version whose build artifact its Bun and uWS jobs download (falling back to the unit-test version when all artifact names are matrixed). The integration check names carry the Node version, so moving the PR version means changing the ruleset's required checks in the same release. `.node-version` and `.nvmrc` hold only the major, and every `setup-node` step reading `.node-version` sets `check-latest`, so tooling, release and publish jobs run the newest stable 26.x rather than the runner's cached one or a prerelease. +Integration build artifacts exclude `node_modules/harper`, the development self-link, to avoid following it into the checkout. Each consuming lane's existing npm install recreates the link and bin shims after download. The strict-peer lockfile compatibility gate checks the scoped framework policy without hiding unrelated peer conflicts. + ### Unit-test fixtures don't spend the `lmdb` shard on one durable commit per row `test:unit:lmdb` re-runs `unitTests/resources`, `unitTests/apiTests` and `unitTests/bin` on LMDB, where every awaited commit waits on a disk sync. A fixture that seeds or clears one row per transaction costs seconds on the default engine and minutes on LMDB, and a runner with slow fsync multiplies exactly that part. `subscriptionPreviousCountScanBound.test.js` (harper#2826), `subscriptionSuperseded.test.js` (harper#2887) and `calibrationStore.test.js` each pushed the step past its cap this way. Nothing enforces this; the step's `timeout-minutes` is the only detector. diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d4b8c79ff3..73dba263b6 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -120,6 +120,7 @@ jobs: dist/ static/ node_modules/ + !node_modules/harper package.json retention-days: 7 @@ -257,6 +258,7 @@ jobs: dist/ static/ node_modules/ + !node_modules/harper package.json retention-days: 1 diff --git a/.github/workflows/lockfile-npm-compat.yml b/.github/workflows/lockfile-npm-compat.yml index c890d7e024..62c49a1b5b 100644 --- a/.github/workflows/lockfile-npm-compat.yml +++ b/.github/workflows/lockfile-npm-compat.yml @@ -41,4 +41,4 @@ jobs: timeout-minutes: 3 run: | echo "npm $(npm -v)" - npm ci --ignore-scripts --dry-run --no-audit --no-fund + npm ci --ignore-scripts --strict-peer-deps --dry-run --no-audit --no-fund diff --git a/build-tools/DESIGN.md b/build-tools/DESIGN.md index d57e24d694..f1379a9662 100644 --- a/build-tools/DESIGN.md +++ b/build-tools/DESIGN.md @@ -12,6 +12,8 @@ Index of every design note: [DESIGN.md](../DESIGN.md). `build.sh` installs the repository lock with `npm ci`, builds Harper and Studio, and stages npm's own source-file selection. Only the staged manifest declares `bundleDependencies`; a root `npm pack` remains a source package and cannot capture the development dependency tree. `prepareBundle` copies the locked, production-reachable JavaScript closure at its original nested/hoisted paths, leaving development-only entries behind. Existing destinations must carry this preparer's ownership marker before they can be cleared. npm 10/11's array and npm 12's name-keyed pack output are both accepted. `checkBundle` validates both the stage and the extracted final archive against the checkout's lock, including copied dependency declarations and peer optionality. The tarball moves from staging to its release filename only after validation; rebuilding removes an older release archive first. `bundleDependencies.test.mjs` covers a real pack → consumer install → offline `npm ci`, nested copies, native rejection and deliberate version/declaration corruption. +The local development Harper peer is a self-link for the integration framework, not a production dependency. Staging removes its framework-scoped override along with `devDependencies`; otherwise `$harper` would reference a removed declaration. Archive inspection rejects that leftover override, and the pack/consumer tests verify that unrelated overrides survive. + Native-containing roots remain ordinary dependencies with exact versions from the lock in the release manifest. Rewriting a non-registry source spec is rejected rather than changing the package's provenance. This includes JavaScript wrappers (`msgpackr`, `cbor-x`, `structon`) and the Bare-runtime optional trees of `tar-fs` and `tar-stream`. `ws` also stays external with its optional native `bufferutil` accelerator so consumers can override the transport independently; optional-only peers are not followed by the bundle planner. Shared direct packages (`@harperfast/extended-iterable`, `ordered-binary`, `weak-lru-cache`) also stay unbundled. The locked storage engines must still declare every module in the alignment contract; declaration changes fail packaging and require updating that contract. Installation checks compare the resolution paths of the storage engines' shared iterables/encoder modules with Harper's, including a consumer layout where npm hoists native roots out of Harper's directory. Negative fixtures reject nested and linked private encoder copies. `SKIP` itself is already a global Symbol; duplicate module copies do not create distinct sentinels. The same helper supports harper-pro’s `re2` and `@datadog/pprof` native roots as ordinary exact-pinned dependencies. Neither is present in core’s lock, so these exclusions do not change core’s bundle. New pro-only native roots require an explicit policy update here rather than a separate copier or CLI policy parameter. diff --git a/build-tools/bundleDependencies.ts b/build-tools/bundleDependencies.ts index c6f34e7d90..3fb8a18063 100644 --- a/build-tools/bundleDependencies.ts +++ b/build-tools/bundleDependencies.ts @@ -25,6 +25,7 @@ type Package = { peerDependenciesMeta?: Record; bundleDependencies?: string[]; devDependencies?: Record; + overrides?: Record>; files?: string[]; scripts?: Record; os?: string[]; @@ -258,6 +259,7 @@ export function prepareBundle(source: string, destination: string) { rmSync(join(stage, 'npm-shrinkwrap.json'), { force: true }); const manifest: Package = readJson(join(stage, 'package.json')); delete manifest.devDependencies; + delete manifest.overrides?.['@harperfast/integration-testing']; manifest.bundleDependencies = plan.roots; manifest.files = manifest.files?.filter((file) => file !== 'npm-shrinkwrap.json'); for (const [name, version] of Object.entries(plan.external)) { @@ -320,6 +322,9 @@ export function checkBundle(root: string, lockFile: string, installed = false) { if (manifest.name !== lock.packages[''].name || manifest.version !== lock.packages[''].version) { throw new Error('Release identity differs from package-lock.json'); } + if (manifest.overrides?.['@harperfast/integration-testing']) { + throw new Error('Release must not contain the development host override'); + } if (existsSync(join(root, 'npm-shrinkwrap.json'))) throw new Error('Release must not contain npm-shrinkwrap.json'); if (JSON.stringify(manifest.bundleDependencies?.toSorted()) !== JSON.stringify(plan.roots.toSorted())) { throw new Error('Release bundleDependencies differ from the production bundle plan'); diff --git a/dependencies.md b/dependencies.md index 255a6836e8..e32626f44d 100644 --- a/dependencies.md +++ b/dependencies.md @@ -304,4 +304,10 @@ This is the inverse of the entries below — a dependency we take deliberate ste - Need for usage: Released fulltext 0.5.0, 0.7.0 and 0.8.0 write genuine old checkpoints; the current engine must rebuild incompatible indexes or reuse compatible ones, and prior engines verify safe reopen or downgrade refusal. Source-record IDs and checkpoints are generated during the tests, so static index files do not replace these checks. - Installation boundary: Harper directly pins the current optional engine; all non-development fulltext entries in the lockfile select 0.9.0. Historical migration aliases are removed from the root dependencies. Separate exact-pinned manifests and integrity-checked lockfiles live under `unitTests/resources/fixtures/fulltext-upgrade/`; the native CI job provisions all three outside the checkout, and existing integration lanes provision only 0.5.0. The previous-engine child exits before the current engine accesses the same index. Production code never imports historical engines or relaxes current ABI validation. - Local qualification: Run `npm run test:fulltext:install-previous -- /absolute/scratch/fulltext-upgrade`, then set `HARPER_FULLTEXT_UPGRADE_ROOT` to that directory when running the native lifecycle, schema-upgrade, crash-recovery or deployed full-text integration suites. Ordinary resource tests skip historical migration cases when the variable is absent; an explicit but invalid installation fails. CI supplies it; the native unit job also forbids pending tests. -- Cost and removal: The dedicated job installs three root packages and their supported platform prebuilds. Integration lanes install only the 0.5.0 fixture; ordinary installs do not provision these migration fixtures. The pre-existing `@harperfast/integration-testing` development peer still brings Harper 5.3.1 and its fulltext 0.5.0 package into the development lock graph. That separate framework dependency cleanup is tracked in [Implement full-text generation lifecycle, rebuild, eviction, backup, and status](https://github.com/HarperFast/harper/issues/2510); do not force its older Harper runtime onto the current native ABI. Renovate leaves fixture versions fixed; remove each manifest, lockfile and matching coverage when migration from that version is no longer supported. +- Cost and removal: The dedicated job installs three root packages and their supported platform prebuilds. Integration lanes install only the 0.5.0 fixture; ordinary installs do not provision these migration fixtures. Renovate leaves fixture versions fixed; remove each manifest, lockfile and matching coverage when migration from that version is no longer supported. + +## Local Harper development peer + +- Need for usage: `@harperfast/integration-testing` requires a Harper peer. npm does not treat the root checkout as an installed peer, so it otherwise installs a second released Harper and its older fulltext dependency. The development-only `harper: file:.` link supplies this checkout; the framework-scoped `$harper` override accepts the local prerelease without disabling other peer validation. +- Boundaries: Build before running integration tests or `npx harper`. CI omits the self-link from dependency artifacts and recreates it with npm after download. Release packaging removes development dependencies and this scoped override; production consumers receive neither the link nor the framework. Historical engines remain isolated migration fixtures. +- Eventual removal: Remove the self dependency and scoped override together when a released framework makes its Harper peer optional and the explicit/local binary route is qualified. Do not override fulltext inside an older Harper runtime. diff --git a/integrationTests/README.md b/integrationTests/README.md index 7effdf6837..4c461b62bf 100644 --- a/integrationTests/README.md +++ b/integrationTests/README.md @@ -34,6 +34,10 @@ Integration tests require a built distribution of Harper. Run this before execut npm run build ``` +The development dependency graph links `node_modules/harper` to this checkout, so `npx harper` uses its built CLI. The framework can select another build through `harperBinPath` or `HARPER_INTEGRATION_TEST_INSTALL_SCRIPT`; CI pins the latter to `dist/bin/harper.js`. Keep the build current before running tests. + +Use `npm ci` to clean dependencies. The local Harper link is a symlink (a junction on Windows); do not use cleanup or copy tools that follow directory links back into the checkout. + ### Loopback Addresses Running tests concurrently requires multiple loopback addresses. Linux systems have these enabled by default; macOS and Windows do not. diff --git a/package-lock.json b/package-lock.json index 244db0c27b..8a03658158 100644 --- a/package-lock.json +++ b/package-lock.json @@ -122,6 +122,7 @@ "compression-1.7": "npm:compression@1.8.2", "eventsource": "^5.0.0", "globals": "^17.0.0", + "harper": "file:.", "intercept-stdout": "0.1.2", "mkcert": "^3.2.0", "mocha": "^12.0.0", @@ -3331,111 +3332,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@lmdb/lmdb-darwin-arm64": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/@lmdb/lmdb-darwin-arm64/-/lmdb-darwin-arm64-3.5.6.tgz", - "integrity": "sha512-mY5FG4TjPAkY4P0w+OhHaUka5mDh2TX2WKYIwuKzJ1zeW3VvRgxdam/lGJTquI+bthTx5CSHDW+BAQCnNAzkEA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "peer": true - }, - "node_modules/@lmdb/lmdb-darwin-x64": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/@lmdb/lmdb-darwin-x64/-/lmdb-darwin-x64-3.5.6.tgz", - "integrity": "sha512-foa+pwitysO8k+xhs7psBFfTKnVgR69NlZRRTHaFVDqphh7AdGpLeyRzKw/ofatr/sN6TiHRRW6mmop0ZrrppQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "peer": true - }, - "node_modules/@lmdb/lmdb-linux-arm": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-arm/-/lmdb-linux-arm-3.5.6.tgz", - "integrity": "sha512-QR4YRyR5h5Z8eGXrNQjiyo2NNDfqi3tCc9dQG5Is1blCt+qWw1ZoBWhlWAr5d+jshkifMIJjVHzHGKbkKzF8Tw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true - }, - "node_modules/@lmdb/lmdb-linux-arm64": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-arm64/-/lmdb-linux-arm64-3.5.6.tgz", - "integrity": "sha512-HmiyFFdJa38s1heCMSooSPaBSFTHJ3C+ERPp28xAPlDX1YiALJVOgbry065nXd8Y7KISWjnw05zpG1RX8IfftA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true - }, - "node_modules/@lmdb/lmdb-linux-x64": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-x64/-/lmdb-linux-x64-3.5.6.tgz", - "integrity": "sha512-ADzCuCF2cTNiX9kDScqcz1fjnAkxPpQNneV3KFTdV3wWtVlI2sTGzySoMTgDpinkMMFj1NTJlxA6XR8fwc4hlA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true - }, - "node_modules/@lmdb/lmdb-win32-arm64": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/@lmdb/lmdb-win32-arm64/-/lmdb-win32-arm64-3.5.6.tgz", - "integrity": "sha512-J7A9aEQsQiv0TYtBGL7NDIPp2lOS8nnl+zm4sWZm1xlsTTaQ4PgD096Adzdrk27rw3UxCkDXdCUa4ax41oztBQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "peer": true - }, - "node_modules/@lmdb/lmdb-win32-x64": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/@lmdb/lmdb-win32-x64/-/lmdb-win32-x64-3.5.6.tgz", - "integrity": "sha512-1g7G0knRX2iV/voDu54yxrGqw5Dk0w2oIYb7dgJq8IkOi+m7wbD8Q3QpPFjh0C01G58S88dqGn03len6UPCXsg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "peer": true - }, "node_modules/@lukeed/ms": { "version": "2.0.2", "license": "MIT", @@ -6568,32 +6464,6 @@ "node": ">= 0.4" } }, - "node_modules/duplexify": { - "version": "4.1.3", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "end-of-stream": "^1.4.1", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1", - "stream-shift": "^1.0.2" - } - }, - "node_modules/duplexify/node_modules/readable-stream": { - "version": "3.6.2", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/easy-ocsp": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/easy-ocsp/-/easy-ocsp-1.3.2.tgz", @@ -7904,950 +7774,172 @@ "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-package-type": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", - "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/glob": { - "version": "13.0.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", - "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", - "license": "BlueOak-1.0.0", - "dependencies": { - "minimatch": "^10.2.2", - "minipass": "^7.1.3", - "path-scurry": "^2.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/glob-parent": { - "version": "6.0.2", - "dev": true, - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.3" - }, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/globals": { - "version": "17.13.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-17.13.0.tgz", - "integrity": "sha512-RwMTC61u7hrG3ZJMkZQOK4JLJrKS8QtoTii63EmWvFXHqycY9FObBJQCbsLxSO8kjKhWE5kV1GC+Vb1g3RI0Zg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "license": "ISC" - }, - "node_modules/graphemer": { - "version": "1.4.0", - "dev": true, - "license": "MIT" - }, - "node_modules/graphql": { - "version": "16.14.2", - "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.14.2.tgz", - "integrity": "sha512-Chq1s4CY7jmh8gO2qvLIJyfCDIN+EHLFW/9iShnp1z8FjBQMoodWP1kDC36VAMXXIvAjj4ARa7ntfAV2BrjsbA==", - "license": "MIT", - "engines": { - "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" - } - }, - "node_modules/graphql-http": { - "version": "1.23.1", - "resolved": "https://registry.npmjs.org/graphql-http/-/graphql-http-1.23.1.tgz", - "integrity": "sha512-h6AcykL7+xp22QvOnK0fu1eHk9AZG1ngF71rk0ZGuGupnjm1yqXRFrZ/eMv4Q1FTCWILJ5o6hVAxzt3S8/wPHQ==", - "license": "MIT", - "workspaces": [ - "implementations/**/*" - ], - "engines": { - "node": ">=12" - }, - "peerDependencies": { - "graphql": ">=0.11 <=17" - } - }, - "node_modules/gunzip-maybe": { - "version": "1.4.2", - "license": "MIT", - "dependencies": { - "browserify-zlib": "^0.1.4", - "is-deflate": "^1.0.0", - "is-gzip": "^1.0.0", - "peek-stream": "^1.1.0", - "pumpify": "^1.3.3", - "through2": "^2.0.3" - }, - "bin": { - "gunzip-maybe": "bin.js" - } - }, - "node_modules/gunzip-maybe/node_modules/duplexify": { - "version": "3.7.1", - "license": "MIT", - "dependencies": { - "end-of-stream": "^1.0.0", - "inherits": "^2.0.1", - "readable-stream": "^2.0.0", - "stream-shift": "^1.0.0" - } - }, - "node_modules/gunzip-maybe/node_modules/pump": { - "version": "2.0.1", - "license": "MIT", - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, - "node_modules/gunzip-maybe/node_modules/pumpify": { - "version": "1.5.1", - "license": "MIT", - "dependencies": { - "duplexify": "^3.6.0", - "inherits": "^2.0.3", - "pump": "^2.0.0" - } - }, - "node_modules/harper": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/harper/-/harper-5.3.1.tgz", - "integrity": "sha512-ESiMdD5H5AZIjcMJ6ocMj9i78DpLtdIM46fLJJQfRmeoEjq+PFX2ifrmNMlDQI74qXM9nF65ygMNsxB5zMabmw==", - "dev": true, - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@endo/static-module-record": "^1.1.2", - "@fastify/autoload": "^6.3.1", - "@fastify/compress": "^8.3.1", - "@fastify/cors": "^11.2.0", - "@fastify/static": "^9.1.3", - "@harperfast/extended-iterable": "1.0.3", - "@harperfast/rocksdb-js": "2.10.0", - "@harperfast/skills": "^1.10.8", - "@inquirer/confirm": "^6.3.2", - "@inquirer/input": "^5.1.6", - "@inquirer/password": "^5.2.2", - "@inquirer/select": "^5.2.5", - "@turf/area": "6.5.0", - "@turf/boolean-contains": "6.5.0", - "@turf/boolean-disjoint": "6.5.0", - "@turf/boolean-equal": "6.5.0", - "@turf/circle": "6.5.0", - "@turf/difference": "6.5.0", - "@turf/distance": "6.5.0", - "@turf/helpers": "6.5.0", - "@turf/length": "6.5.0", - "alasql": "4.19.1", - "amaro": "^1.1.8", - "argon2": "0.45.1", - "asn1js": "3.0.10", - "busboy": "^1.6.0", - "cbor-x": "1.6.6", - "chalk": "4.1.2", - "chokidar": "^4.0.3", - "cli-progress": "3.12.0", - "clone": "2.1.2", - "dotenv": "^16.4.7", - "easy-ocsp": "1.3.2", - "fast-glob": "3.3.3", - "fastify": "^5.8.2", - "fastify-plugin": "^5.1.0", - "fs-extra": "11.4.0", - "graphql": "^16.10.0", - "graphql-http": "^1.22.4", - "gunzip-maybe": "1.4.2", - "human-readable-ids": "1.0.4", - "is-number": "7.0.0", - "joi": "17.13.8", - "json-bigint-fixes": "1.1.0", - "jsonata": "1.8.9", - "jsonwebtoken": "9.0.3", - "lmdb": "3.5.6", - "lodash": "^4.17.23", - "mathjs": "11.12.0", - "micromatch": "^4.0.8", - "minimist": "1.2.8", - "moment": "2.31.0", - "mqtt-packet": "~9.0.1", - "msgpackr": "2.1.0", - "needle": "3.5.0", - "node-forge": "^1.3.1", - "node-stream-zip": "1.16.0", - "normalize-path": "^3.0.0", - "ora": "8.2.0", - "ordered-binary": "1.6.2", - "papaparse": "5.7.0", - "passport": "0.7.0", - "passport-http": "0.3.0", - "passport-local": "1.0.0", - "pino": "8.21.0", - "pkijs": "3.4.1", - "properties-reader": "2.3.0", - "recursive-iterator": "3.3.0", - "semver": "7.8.5", - "send": "^1.2.0", - "ses": "^1.15.0", - "stream-chain": "2.2.5", - "stream-json": "1.9.1", - "structon": "1.1.0", - "systeminformation": "^5.31.4", - "tar-fs": "^3.1.2", - "tar-stream": "3.2.0", - "ulidx": "0.5.0", - "uuid": "11.1.1", - "validate.js": "0.13.1", - "weak-lru-cache": "^1.2.2", - "ws": "8.21.3", - "yaml": "2.9.1" - }, - "bin": { - "harper": "dist/bin/harper.js" - }, - "engines": { - "node": "^22.18.0 || >=24.0.0" - }, - "optionalDependencies": { - "@harperfast/fulltext": "0.5.0", - "@harperfast/hnsw": "0.4.0", - "bufferutil": "4.1.0", - "segfault-handler": "1.3.0", - "utf-8-validate": "5.0.10" - }, - "peerDependencies": { - "@aws-sdk/client-bedrock-runtime": "^3.0.0", - "@aws-sdk/client-s3": "^3.0.0", - "@aws-sdk/lib-storage": "^3.0.0", - "uWebSockets.js": "20.68.0" - }, - "peerDependenciesMeta": { - "@aws-sdk/client-bedrock-runtime": { - "optional": true - }, - "@aws-sdk/client-s3": { - "optional": true - }, - "@aws-sdk/lib-storage": { - "optional": true - }, - "uWebSockets.js": { - "optional": true - } - } - }, - "node_modules/harper/node_modules/@fastify/compress": { - "version": "8.3.1", - "resolved": "https://registry.npmjs.org/@fastify/compress/-/compress-8.3.1.tgz", - "integrity": "sha512-BUpItLr6MUX9e9ukg5Y6xekyA/7pBFG8QWtFCrUDm9ctoBc3R2/nA16yOaOWtVoccpXGjdDEYA/MxAb5+8cxag==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "peer": true, - "dependencies": { - "@fastify/accept-negotiator": "^2.0.0", - "fastify-plugin": "^5.0.0", - "mime-db": "^1.52.0", - "minipass": "^7.0.4", - "peek-stream": "^1.1.3", - "pump": "^3.0.0", - "pumpify": "^2.0.1", - "readable-stream": "^4.5.2" - } - }, - "node_modules/harper/node_modules/@fastify/static": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/@fastify/static/-/static-9.3.0.tgz", - "integrity": "sha512-9YMYRpCOtMBrqKYWcqiw7ykOrn4D0jogHpJrFS0KGeSuOwzKMM5/mjj7B0CFLVoQ6htqKYw//Zs7APn9DBq05w==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "peer": true, - "dependencies": { - "@fastify/accept-negotiator": "^2.0.0", - "@fastify/send": "^4.0.0", - "content-disposition": "^1.0.1", - "fastify-plugin": "^6.0.0", - "fastq": "^1.17.1", - "glob": "^13.0.0" - } - }, - "node_modules/harper/node_modules/@fastify/static/node_modules/fastify-plugin": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-6.0.0.tgz", - "integrity": "sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "peer": true - }, - "node_modules/harper/node_modules/@harperfast/extended-iterable": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@harperfast/extended-iterable/-/extended-iterable-1.0.3.tgz", - "integrity": "sha512-sSAYhQca3rDWtQUHSAPeO7axFIUJOI6hn1gjRC5APVE1a90tuyT8f5WIgRsFhhWA7htNkju2veB9eWL6YHi/Lw==", - "dev": true, - "license": "Apache-2.0", - "peer": true - }, - "node_modules/harper/node_modules/@harperfast/fulltext": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext/-/fulltext-0.5.0.tgz", - "integrity": "sha512-rH2Q3LJLHPhxi8muzPLg00STbO7RFIqAgLA3tpBzSRdsJVuCV9ZMO2StlSZnUAPMnF5wF9SieuhHT9wi1vINBQ==", - "dev": true, - "license": "Apache-2.0", - "optional": true, - "peer": true, - "engines": { - "node": "^22.18.0 || >=24.0.0" - }, - "optionalDependencies": { - "@harperfast/fulltext-darwin-arm64": "0.5.0", - "@harperfast/fulltext-linux-arm64-gnu": "0.5.0", - "@harperfast/fulltext-linux-x64-gnu": "0.5.0", - "@harperfast/fulltext-win32-x64-msvc": "0.5.0" - } - }, - "node_modules/harper/node_modules/@harperfast/fulltext-darwin-arm64": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-darwin-arm64/-/fulltext-darwin-arm64-0.5.0.tgz", - "integrity": "sha512-0OtKonopmXaP0/9wvpCuo/sklNxOEiQWnVf7aUsthH0B3sKUYT7HGnGtYkzqqUxdsFNlHPBr3IShyrWx9nAWYQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/harper/node_modules/@harperfast/fulltext-linux-arm64-gnu": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-arm64-gnu/-/fulltext-linux-arm64-gnu-0.5.0.tgz", - "integrity": "sha512-QfNlYUntkal56jZejQ8cbSj8qvUlYv21V2j9xVctkFv+gmfn892kdU4oCg7dR7Qift8/huKUOyL/eGO0qYbgGw==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/harper/node_modules/@harperfast/fulltext-linux-x64-gnu": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-x64-gnu/-/fulltext-linux-x64-gnu-0.5.0.tgz", - "integrity": "sha512-+g5LMXhtODen+pGgmyTxcCuUAlk2NTeFOvr6duqUBVx542Hd/uDqPC9tUMjL12LJ9ci3LFqGwgW3FkxGedT7KA==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/harper/node_modules/@harperfast/fulltext-win32-x64-msvc": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-win32-x64-msvc/-/fulltext-win32-x64-msvc-0.5.0.tgz", - "integrity": "sha512-PseSz47BMcIOMQdYUFoAUqAEyeHzceX+QQFun0wVCu/PtERPvtPWzzZxV4ofySzz+c9FkJHTSDu4XbAZhkaqkA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/harper/node_modules/@harperfast/rocksdb-js": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js/-/rocksdb-js-2.10.0.tgz", - "integrity": "sha512-o9oCwMMZfRK0jvHA4jmKU5g4+AGm7q/gas3ZJgas09AkNu0YZVnmun0zqVg6kZCzZEsUdjMUKdemLeI/RnSTVg==", - "dev": true, - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@harperfast/extended-iterable": "^1.0.3", - "msgpackr": "^2.0.6", - "ordered-binary": "^1.6.1" - }, - "bin": { - "rocksdb-js": "bin/rocksdb-js.mjs" - }, - "engines": { - "node": "^22.18.0 || >=24.0.0" - }, - "optionalDependencies": { - "@harperfast/rocksdb-js-darwin-arm64": "2.10.0", - "@harperfast/rocksdb-js-darwin-x64": "2.10.0", - "@harperfast/rocksdb-js-linux-arm64-glibc": "2.10.0", - "@harperfast/rocksdb-js-linux-arm64-musl": "2.10.0", - "@harperfast/rocksdb-js-linux-x64-glibc": "2.10.0", - "@harperfast/rocksdb-js-linux-x64-musl": "2.10.0", - "@harperfast/rocksdb-js-win32-arm64": "2.10.0", - "@harperfast/rocksdb-js-win32-x64": "2.10.0" - } - }, - "node_modules/harper/node_modules/@harperfast/rocksdb-js-darwin-arm64": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-arm64/-/rocksdb-js-darwin-arm64-2.10.0.tgz", - "integrity": "sha512-VFyg2CcW3k5e7Qt65BPVZ9lvQz9FWCdNdwJJPfAScXAPrpyzklj9Qp3w/bTgW41+6qQbYiEHGUa5jK7qqMsBOA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/harper/node_modules/@harperfast/rocksdb-js-darwin-x64": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-x64/-/rocksdb-js-darwin-x64-2.10.0.tgz", - "integrity": "sha512-OccEC4uWEzX9S1kzUFocBcEUGql6m4CVZlqkQTMZMS4RqA6O7YNNKb/smaiKISmPaS1vgIbyI0dmo08gypkZ/g==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/harper/node_modules/@harperfast/rocksdb-js-linux-arm64-glibc": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-glibc/-/rocksdb-js-linux-arm64-glibc-2.10.0.tgz", - "integrity": "sha512-W1SrRk/ijpVPkmHPCKrfEd6yrgK6jgadS45Q1bhlGY3LJj1fWGx0CodMtZQ9rOiyTi7RLaB+IS/wXkjgV6puog==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/harper/node_modules/@harperfast/rocksdb-js-linux-arm64-musl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-musl/-/rocksdb-js-linux-arm64-musl-2.10.0.tgz", - "integrity": "sha512-e7pI3wbOGyV5YQtEItuL/RmzRi2b0ysGWvooOSEx/fDjGidirHcqtL4ejydwdXS29Em5+iDkgdHyeKZAopiV7w==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/harper/node_modules/@harperfast/rocksdb-js-linux-x64-glibc": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-glibc/-/rocksdb-js-linux-x64-glibc-2.10.0.tgz", - "integrity": "sha512-8Ap0n+5pcG2/oX61Gfi88CmNDcKg3EU7ektpBsEgdXmvzkMb9E9hwVu2zDIMezG3pLmddEvmIlN/qOYwqNW3wA==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/harper/node_modules/@harperfast/rocksdb-js-linux-x64-musl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-musl/-/rocksdb-js-linux-x64-musl-2.10.0.tgz", - "integrity": "sha512-iXzNINutMiSKq8QcGm2gLrtmegROcU+CMl67xjBaIwerb8x/qYGISecw04L9BlcMbX/pSYZpBZJfC6j/dJZunw==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/harper/node_modules/@harperfast/rocksdb-js-win32-arm64": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-arm64/-/rocksdb-js-win32-arm64-2.10.0.tgz", - "integrity": "sha512-BcR26Y9ScJ0sycm+HJ8xaepH1vkzlQ1qWhm0803szb0vIfxPAKGVn3ZaybLQW696ssgfD+taKz23rEalRTmrpg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/harper/node_modules/@harperfast/rocksdb-js-win32-x64": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-x64/-/rocksdb-js-win32-x64-2.10.0.tgz", - "integrity": "sha512-l/jNPBiySXVfTMNndE4C5ZpyAZEHag0NZgkDaHABlEUuTrbDwb7RmOkdLusJaAHS1Tl8sguk3W8iy8ieU+0mHw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": "^22.18.0 || >=24.0.0" - } - }, - "node_modules/harper/node_modules/alasql": { - "version": "4.19.1", - "resolved": "https://registry.npmjs.org/alasql/-/alasql-4.19.1.tgz", - "integrity": "sha512-SyI0oN/IU31/Lv8q4xmTo4Oo+sWKgTM9bDvJ+7SVdAh/tf100ppF/UqZKlDjeKyr+/JfWSFGft14HmQpXzYLOg==", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "cross-fetch": "4.1.0", - "yargs": "16" - }, - "bin": { - "alasql": "bin/alasql-cli.js" - }, - "engines": { - "node": ">=15" - }, - "optionalDependencies": { - "react-native-fs": "^2.20.0" - } - }, - "node_modules/harper/node_modules/asn1js": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/asn1js/-/asn1js-3.0.10.tgz", - "integrity": "sha512-S2s3aOytiKdFRdulw2qPE51MzjzVOisppcVv7jVFR+Kw0kxwvFrDcYA0h7Ndqbmj0HkMIXYWaoj7fli8kgx1eg==", - "dev": true, - "license": "BSD-3-Clause", - "peer": true, - "dependencies": { - "pvtsutils": "^1.3.6", - "pvutils": "^1.1.5", - "tslib": "^2.8.1" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/harper/node_modules/buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "peer": true, - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/harper/node_modules/cbor-x": { - "version": "1.6.6", - "resolved": "https://registry.npmjs.org/cbor-x/-/cbor-x-1.6.6.tgz", - "integrity": "sha512-8QiD9PGOxyQHo7s2pzwTBH6lTjqekxPdl9Aq6fXvZgCuCJHOht1puDEA/fTr6mciB76c+M+Gi0qT2i1a4pm4Wg==", - "dev": true, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", "license": "MIT", + "optional": true, "peer": true, - "optionalDependencies": { - "cbor-extract": "^2.2.2" + "engines": { + "node": ">=8.0.0" } }, - "node_modules/harper/node_modules/chokidar": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", - "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", - "dev": true, + "node_modules/get-proto": { + "version": "1.0.1", "license": "MIT", - "peer": true, "dependencies": { - "readdirp": "^4.0.1" + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" }, "engines": { - "node": ">= 14.16.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" + "node": ">= 0.4" } }, - "node_modules/harper/node_modules/fs-extra": { - "version": "11.4.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.0.tgz", - "integrity": "sha512-EQsFzMUJkCKGr1ePqlYADkIUmHW1s3ZXr5Yqy6wbGrfUCphpl2maM/kyOIRA2HpP3AaFQTZXD4ldjek+nccddA==", - "dev": true, - "license": "MIT", - "peer": true, + "node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" }, "engines": { - "node": ">=14.14" + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/harper/node_modules/lmdb": { - "version": "3.5.6", - "resolved": "https://registry.npmjs.org/lmdb/-/lmdb-3.5.6.tgz", - "integrity": "sha512-j3uE8ReKNyUWDjhfEFSJqE/1DLtfTR5Z8yFzVHvBjAk37wNg7HdScjcv8ttPHRvrdgPQMPWxFFI0SsdBzI5lBw==", + "node_modules/glob-parent": { + "version": "6.0.2", "dev": true, - "hasInstallScript": true, - "license": "MIT", - "peer": true, + "license": "ISC", "dependencies": { - "@harperfast/extended-iterable": "^1.0.3", - "msgpackr": "^1.11.2", - "node-addon-api": "^6.1.0", - "node-gyp-build-optional-packages": "5.2.2", - "ordered-binary": "^1.5.3", - "weak-lru-cache": "^1.2.2" - }, - "bin": { - "download-lmdb-prebuilds": "bin/download-prebuilds.js" + "is-glob": "^4.0.3" }, - "optionalDependencies": { - "@lmdb/lmdb-darwin-arm64": "3.5.6", - "@lmdb/lmdb-darwin-x64": "3.5.6", - "@lmdb/lmdb-linux-arm": "3.5.6", - "@lmdb/lmdb-linux-arm64": "3.5.6", - "@lmdb/lmdb-linux-x64": "3.5.6", - "@lmdb/lmdb-win32-arm64": "3.5.6", - "@lmdb/lmdb-win32-x64": "3.5.6" + "engines": { + "node": ">=10.13.0" } }, - "node_modules/harper/node_modules/lmdb/node_modules/msgpackr": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.12.1.tgz", - "integrity": "sha512-4EUH9tQHnMmEgzW/MdAP0KIfa1T9AF+htl0ffe2n5vb2EKn9y2co8ccpgWko6S52Jy1PQZKwRnx5/KkYjtd9MQ==", + "node_modules/globals": { + "version": "17.13.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.13.0.tgz", + "integrity": "sha512-RwMTC61u7hrG3ZJMkZQOK4JLJrKS8QtoTii63EmWvFXHqycY9FObBJQCbsLxSO8kjKhWE5kV1GC+Vb1g3RI0Zg==", "dev": true, "license": "MIT", - "peer": true, - "optionalDependencies": { - "msgpackr-extract": "^3.0.2" + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/harper/node_modules/msgpackr": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-2.1.0.tgz", - "integrity": "sha512-p/pBCVO63CsvvpkomUnNNag6+n38rULuDA6HHe70o2gtC8ODI52foF/4ko2qQcp6OiErJXTmrZeXmsGGHsIQNQ==", - "dev": true, + "node_modules/gopd": { + "version": "1.2.0", "license": "MIT", - "peer": true, - "optionalDependencies": { - "msgpackr-extract": "^3.0.4" + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/harper/node_modules/node-addon-api": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", - "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", - "dev": true, - "license": "MIT", - "peer": true + "node_modules/graceful-fs": { + "version": "4.2.11", + "license": "ISC" }, - "node_modules/harper/node_modules/node-gyp-build-optional-packages": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", - "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "node_modules/graphemer": { + "version": "1.4.0", "dev": true, + "license": "MIT" + }, + "node_modules/graphql": { + "version": "16.14.2", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.14.2.tgz", + "integrity": "sha512-Chq1s4CY7jmh8gO2qvLIJyfCDIN+EHLFW/9iShnp1z8FjBQMoodWP1kDC36VAMXXIvAjj4ARa7ntfAV2BrjsbA==", "license": "MIT", - "peer": true, - "dependencies": { - "detect-libc": "^2.0.1" - }, - "bin": { - "node-gyp-build-optional-packages": "bin.js", - "node-gyp-build-optional-packages-optional": "optional.js", - "node-gyp-build-optional-packages-test": "build-test.js" + "engines": { + "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } }, - "node_modules/harper/node_modules/ordered-binary": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/ordered-binary/-/ordered-binary-1.6.2.tgz", - "integrity": "sha512-BY+qgDArmYpvvmij141xbkB4Cw8RFLBQM5Zj4MODFb1SyXBRE/xO4Xj/QWbR7aUEBxbkjuNdsFVCL2TmbYrlNg==", - "dev": true, + "node_modules/graphql-http": { + "version": "1.23.1", + "resolved": "https://registry.npmjs.org/graphql-http/-/graphql-http-1.23.1.tgz", + "integrity": "sha512-h6AcykL7+xp22QvOnK0fu1eHk9AZG1ngF71rk0ZGuGupnjm1yqXRFrZ/eMv4Q1FTCWILJ5o6hVAxzt3S8/wPHQ==", "license": "MIT", - "peer": true - }, - "node_modules/harper/node_modules/pkijs": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/pkijs/-/pkijs-3.4.1.tgz", - "integrity": "sha512-Oo/NZcSWccq8KyoG7gLE9fnltgHns+pNCjCAp/WmjsUySi+sX7y4z4Xqu4fVb42CDHzRPl33fjzT15V1wvcyhA==", - "dev": true, - "license": "BSD-3-Clause", - "peer": true, "workspaces": [ - "website" + "implementations/**/*" ], - "dependencies": { - "@noble/hashes": "1.8.0", - "asn1js": "^3.0.6", - "bytestreamjs": "^2.0.1", - "pvtsutils": "^1.3.6", - "pvutils": "^1.1.3", - "tslib": "^2.8.1" - }, "engines": { - "node": ">=16.0.0" + "node": ">=12" + }, + "peerDependencies": { + "graphql": ">=0.11 <=17" } }, - "node_modules/harper/node_modules/readable-stream": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", - "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", - "dev": true, + "node_modules/gunzip-maybe": { + "version": "1.4.2", "license": "MIT", - "peer": true, "dependencies": { - "abort-controller": "^3.0.0", - "buffer": "^6.0.3", - "events": "^3.3.0", - "process": "^0.11.10", - "string_decoder": "^1.3.0" + "browserify-zlib": "^0.1.4", + "is-deflate": "^1.0.0", + "is-gzip": "^1.0.0", + "peek-stream": "^1.1.0", + "pumpify": "^1.3.3", + "through2": "^2.0.3" }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + "bin": { + "gunzip-maybe": "bin.js" } }, - "node_modules/harper/node_modules/readdirp": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", - "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", - "dev": true, + "node_modules/gunzip-maybe/node_modules/duplexify": { + "version": "3.7.1", "license": "MIT", - "peer": true, - "engines": { - "node": ">= 14.18.0" - }, - "funding": { - "type": "individual", - "url": "https://paulmillr.com/funding/" + "dependencies": { + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" } }, - "node_modules/harper/node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, + "node_modules/gunzip-maybe/node_modules/pump": { + "version": "2.0.1", "license": "MIT", - "peer": true, "dependencies": { - "safe-buffer": "~5.2.0" - } - }, - "node_modules/harper/node_modules/structon": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/structon/-/structon-1.1.0.tgz", - "integrity": "sha512-QveeRWPoVevondkGanTbYjxfPvrPtfSKLUXIt3rUM5EK8p4K4lJceNc0juernn+YlfhpCZDpIcJEr3oZxGcimw==", - "dev": true, - "license": "Apache-2.0", - "peer": true, - "peerDependencies": { - "msgpackr": ">=2.0.1" - }, - "peerDependenciesMeta": { - "msgpackr": { - "optional": true - } + "end-of-stream": "^1.1.0", + "once": "^1.3.1" } }, - "node_modules/harper/node_modules/ws": { - "version": "8.21.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", - "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", - "dev": true, + "node_modules/gunzip-maybe/node_modules/pumpify": { + "version": "1.5.1", "license": "MIT", - "peer": true, - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } + "dependencies": { + "duplexify": "^3.6.0", + "inherits": "^2.0.3", + "pump": "^2.0.0" } }, + "node_modules/harper": { + "resolved": "", + "link": true + }, "node_modules/has-flag": { "version": "4.0.0", "license": "MIT", @@ -12011,17 +11103,6 @@ "once": "^1.3.1" } }, - "node_modules/pumpify": { - "version": "2.0.1", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "duplexify": "^4.1.1", - "inherits": "^2.0.3", - "pump": "^3.0.0" - } - }, "node_modules/punycode": { "version": "2.3.1", "dev": true, @@ -13855,21 +12936,6 @@ "punycode": "^2.1.0" } }, - "node_modules/utf-8-validate": { - "version": "5.0.10", - "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz", - "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==", - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "node-gyp-build": "^4.3.0" - }, - "engines": { - "node": ">=6.14.2" - } - }, "node_modules/utf8": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/utf8/-/utf8-3.0.0.tgz", diff --git a/package.json b/package.json index bfe5deab1a..d143d33d4d 100644 --- a/package.json +++ b/package.json @@ -185,6 +185,7 @@ "compression-1.7": "npm:compression@1.8.2", "eventsource": "^5.0.0", "globals": "^17.0.0", + "harper": "file:.", "intercept-stdout": "0.1.2", "mkcert": "^3.2.0", "mocha": "^12.0.0", @@ -292,6 +293,9 @@ "overrides": { "alasql": { "xlsx": "0.18.5" + }, + "@harperfast/integration-testing": { + "harper": "$harper" } }, "optionalDependencies": { diff --git a/unitTests/build-tools/bundleDependencies.test.mjs b/unitTests/build-tools/bundleDependencies.test.mjs index 0600564a6a..9d33d5f369 100644 --- a/unitTests/build-tools/bundleDependencies.test.mjs +++ b/unitTests/build-tools/bundleDependencies.test.mjs @@ -1,6 +1,7 @@ import assert from 'node:assert'; import { execFileSync } from 'node:child_process'; -import { mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { mkdtempSync, mkdirSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { createRequire, findPackageJSON } from 'node:module'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -215,6 +216,41 @@ describe('portable production dependency bundle', function () { assert.ok(!bundlePlan(optionalOnly).packages.has('node_modules/@types/node')); }); + it('uses this checkout as the integration framework host without a second fulltext engine', () => { + const root = fileURLToPath(new URL('../../', import.meta.url)); + const require = createRequire(join(root, 'package.json')); + const framework = require.resolve('@harperfast/integration-testing'); + assert.strictEqual(realpathSync(findPackageJSON('harper', framework)), realpathSync(join(root, 'package.json'))); + const realLock = JSON.parse(readFileSync(join(root, 'package-lock.json'))); + const manifest = JSON.parse(readFileSync(join(root, 'package.json'))); + for (const [key, entry] of Object.entries(realLock.packages)) { + if (key.endsWith('/harper')) assert.strictEqual(realpathSync(join(root, key)), realpathSync(root)); + if (/node_modules\/@harperfast\/fulltext(?:-|$)/.test(key)) { + assert.strictEqual(entry.version, manifest.optionalDependencies['@harperfast/fulltext']); + } + } + assert.ok(!bundlePlan(realLock).packages.has('node_modules/harper')); + assert.ok(!bundlePlan(realLock).packages.has('node_modules/@harperfast/integration-testing')); + }); + + it('removes development host policy from the release without removing other overrides', () => { + writeSource(); + const file = join(source, 'package.json'); + const manifest = JSON.parse(readFileSync(file)); + manifest.devDependencies.harper = 'file:.'; + manifest.overrides = { '@harperfast/integration-testing': { harper: '$harper' }, 'parent': { child: '1.0.1' } }; + writeFileSync(file, JSON.stringify(manifest)); + symlinkSync(source, join(source, 'node_modules/harper'), 'junction'); + const stage = prepareBundle(source, join(directory, 'stage')); + const packaged = JSON.parse(readFileSync(join(stage, 'package.json'))); + assert.ok(!packaged.devDependencies); + assert.deepStrictEqual(packaged.overrides, { parent: { child: '1.0.1' } }); + assert.deepStrictEqual(JSON.parse(readFileSync(file)), manifest); + packaged.overrides['@harperfast/integration-testing'] = { harper: '$harper' }; + writeFileSync(join(stage, 'package.json'), JSON.stringify(packaged)); + assert.throws(() => checkBundle(stage, join(source, 'package-lock.json')), /development host override/); + }); + it('keeps uWebSockets.js as a dev dependency and optional peer in the real manifest', () => { const manifest = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url))); assert.ok(manifest.devDependencies['uWebSockets.js']); From 4687117d4b515d8f3f6214dcedbb39c1f9e51628 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 12:06:32 -0700 Subject: [PATCH 18/26] Canonicalize packaging fixture paths before npm pack --- unitTests/build-tools/bundleDependencies.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unitTests/build-tools/bundleDependencies.test.mjs b/unitTests/build-tools/bundleDependencies.test.mjs index 9d33d5f369..5e8789644a 100644 --- a/unitTests/build-tools/bundleDependencies.test.mjs +++ b/unitTests/build-tools/bundleDependencies.test.mjs @@ -14,7 +14,7 @@ describe('portable production dependency bundle', function () { let lock; beforeEach(() => { - directory = mkdtempSync(join(tmpdir(), 'harper-bundle-')); + directory = mkdtempSync(join(realpathSync(tmpdir()), 'harper-bundle-')); source = join(directory, 'source'); mkdirSync(source); lock = { From 1478df2b19547a982b67f60c79beb0fda15ae29d Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 12:10:46 -0700 Subject: [PATCH 19/26] Clarify the development override staging regression --- build-tools/DESIGN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools/DESIGN.md b/build-tools/DESIGN.md index f1379a9662..bc6e306091 100644 --- a/build-tools/DESIGN.md +++ b/build-tools/DESIGN.md @@ -12,7 +12,7 @@ Index of every design note: [DESIGN.md](../DESIGN.md). `build.sh` installs the repository lock with `npm ci`, builds Harper and Studio, and stages npm's own source-file selection. Only the staged manifest declares `bundleDependencies`; a root `npm pack` remains a source package and cannot capture the development dependency tree. `prepareBundle` copies the locked, production-reachable JavaScript closure at its original nested/hoisted paths, leaving development-only entries behind. Existing destinations must carry this preparer's ownership marker before they can be cleared. npm 10/11's array and npm 12's name-keyed pack output are both accepted. `checkBundle` validates both the stage and the extracted final archive against the checkout's lock, including copied dependency declarations and peer optionality. The tarball moves from staging to its release filename only after validation; rebuilding removes an older release archive first. `bundleDependencies.test.mjs` covers a real pack → consumer install → offline `npm ci`, nested copies, native rejection and deliberate version/declaration corruption. -The local development Harper peer is a self-link for the integration framework, not a production dependency. Staging removes its framework-scoped override along with `devDependencies`; otherwise `$harper` would reference a removed declaration. Archive inspection rejects that leftover override, and the pack/consumer tests verify that unrelated overrides survive. +The local development Harper peer is a self-link for the integration framework, not a production dependency. Staging removes its framework-scoped override along with `devDependencies`; otherwise `$harper` would reference a removed declaration. Archive inspection rejects that leftover override, and the staging regression verifies that unrelated overrides survive. Native-containing roots remain ordinary dependencies with exact versions from the lock in the release manifest. Rewriting a non-registry source spec is rejected rather than changing the package's provenance. This includes JavaScript wrappers (`msgpackr`, `cbor-x`, `structon`) and the Bare-runtime optional trees of `tar-fs` and `tar-stream`. `ws` also stays external with its optional native `bufferutil` accelerator so consumers can override the transport independently; optional-only peers are not followed by the bundle planner. Shared direct packages (`@harperfast/extended-iterable`, `ordered-binary`, `weak-lru-cache`) also stay unbundled. The locked storage engines must still declare every module in the alignment contract; declaration changes fail packaging and require updating that contract. Installation checks compare the resolution paths of the storage engines' shared iterables/encoder modules with Harper's, including a consumer layout where npm hoists native roots out of Harper's directory. Negative fixtures reject nested and linked private encoder copies. `SKIP` itself is already a global Symbol; duplicate module copies do not create distinct sentinels. From 49cd31ef8e8a8d48d5c306f64b00cbec70cbd46a Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 12:15:36 -0700 Subject: [PATCH 20/26] Remove the development self-link before dependency uploads --- .github/DESIGN.md | 2 +- .github/workflows/integration-tests.yml | 8 ++++++++ dependencies.md | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/DESIGN.md b/.github/DESIGN.md index 35d86b6124..06fb163259 100644 --- a/.github/DESIGN.md +++ b/.github/DESIGN.md @@ -30,7 +30,7 @@ Branch protection requires only the `Unit Test` job, which needs every other job Node 26 is the recommended runtime, so every single-version job runs it, and pull requests run unit tests on 24 and 26 (the `lmdb` shard on 26 alone until harper#3349's segfault is fixed) and integration tests on 26 alone. Pushes to `main` run both on 24 and 26. Node 22 runs nightly and on a manual `all` dispatch, as do the native type-strip legs on `22.18.0` and `24.0.0`, the `engines` floors in `package.json`, which are the oldest runtimes that must run the `.ts` sources directly. The nightly schedule runs only on `main`, so a push to a release branch runs that full set: a patch release still ships to Node 22 users. The cherry-pick job dispatches a release branch's unit tests on Node 26 (or 24 when that branch's workflow predates 26) and its integration tests on the Node version whose build artifact its Bun and uWS jobs download (falling back to the unit-test version when all artifact names are matrixed). The integration check names carry the Node version, so moving the PR version means changing the ruleset's required checks in the same release. `.node-version` and `.nvmrc` hold only the major, and every `setup-node` step reading `.node-version` sets `check-latest`, so tooling, release and publish jobs run the newest stable 26.x rather than the runner's cached one or a prerelease. -Integration build artifacts exclude `node_modules/harper`, the development self-link, to avoid following it into the checkout. Each consuming lane's existing npm install recreates the link and bin shims after download. The strict-peer lockfile compatibility gate checks the scoped framework policy without hiding unrelated peer conflicts. +Integration producers remove `node_modules/harper` before upload and exclude it from artifacts: the globber can still walk an excluded directory link. Each consuming lane's existing npm install recreates the link and bin shims after download. The strict-peer lockfile compatibility gate checks the scoped framework policy without hiding unrelated peer conflicts. ### Unit-test fixtures don't spend the `lmdb` shard on one durable commit per row diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 73dba263b6..102127f14d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -112,6 +112,10 @@ jobs: - name: Build run: npm run build + - name: Remove local development link before upload + timeout-minutes: 1 + run: node -e "require('node:fs').rmSync('node_modules/harper', { recursive: true, force: true })" + - name: Upload build artifacts uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: @@ -250,6 +254,10 @@ jobs: - name: Build run: npm run build + - name: Remove local development link before upload + timeout-minutes: 1 + run: node -e "require('node:fs').rmSync('node_modules/harper', { recursive: true, force: true })" + - name: Upload build artifacts uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: diff --git a/dependencies.md b/dependencies.md index e32626f44d..81c94923ed 100644 --- a/dependencies.md +++ b/dependencies.md @@ -309,5 +309,5 @@ This is the inverse of the entries below — a dependency we take deliberate ste ## Local Harper development peer - Need for usage: `@harperfast/integration-testing` requires a Harper peer. npm does not treat the root checkout as an installed peer, so it otherwise installs a second released Harper and its older fulltext dependency. The development-only `harper: file:.` link supplies this checkout; the framework-scoped `$harper` override accepts the local prerelease without disabling other peer validation. -- Boundaries: Build before running integration tests or `npx harper`. CI omits the self-link from dependency artifacts and recreates it with npm after download. Release packaging removes development dependencies and this scoped override; production consumers receive neither the link nor the framework. Historical engines remain isolated migration fixtures. +- Boundaries: Build before running integration tests or `npx harper`. CI removes the self-link before uploading dependency artifacts and recreates it with npm after download. Release packaging removes development dependencies and this scoped override; production consumers receive neither the link nor the framework. Historical engines remain isolated migration fixtures. - Eventual removal: Remove the self dependency and scoped override together when a released framework makes its Harper peer optional and the explicit/local binary route is qualified. Do not override fulltext inside an older Harper runtime. From e6858213d6fae31c3c016ace689f62312d2ca2a8 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 12:16:06 -0700 Subject: [PATCH 21/26] Quote the pre-upload cleanup as a YAML script block --- .github/workflows/integration-tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 102127f14d..102d7e4858 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -114,7 +114,8 @@ jobs: - name: Remove local development link before upload timeout-minutes: 1 - run: node -e "require('node:fs').rmSync('node_modules/harper', { recursive: true, force: true })" + run: | + node -e "require('node:fs').rmSync('node_modules/harper', { recursive: true, force: true })" - name: Upload build artifacts uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -256,7 +257,8 @@ jobs: - name: Remove local development link before upload timeout-minutes: 1 - run: node -e "require('node:fs').rmSync('node_modules/harper', { recursive: true, force: true })" + run: | + node -e "require('node:fs').rmSync('node_modules/harper', { recursive: true, force: true })" - name: Upload build artifacts uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 From 3f0dd0ff1c341e8b1fe75d33f6e958ec1d038201 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 12:25:55 -0700 Subject: [PATCH 22/26] Align strict-peer and migration dependency design wording --- .github/DESIGN.md | 6 +++--- dependencies.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/DESIGN.md b/.github/DESIGN.md index 06fb163259..1dbf7a08e2 100644 --- a/.github/DESIGN.md +++ b/.github/DESIGN.md @@ -10,9 +10,9 @@ Index of every design note: [DESIGN.md](../DESIGN.md). ## The lock file must pass `npm ci` under npm 10 and npm 11 (`workflows/lockfile-npm-compat.yml`) -npm 11 accepts some lock files that npm 10's `npm ci` rejects as out of sync. Renovate and the Sync Lock File workflow (`.node-version`) write the lock with npm 11, but harper supports Node 22, which ships npm 10. harper#3006 auto-merged such a lock: the root `utf-8-validate` 6.x didn't satisfy the `^5.0.2` optional peer of the `ws@7` copies in the react-native subtree, so npm 10 wants a nested 5.x where npm 11 doesn't. harper#3015 removed that root declaration. `lockfile-npm-compat.yml` runs `npm ci --dry-run` on Node 22 and Node 24 for every PR and `main` push, with no path filter so it can be a required check. +npm 11 accepts some lock files that npm 10's `npm ci` rejects as out of sync. Renovate and the Sync Lock File workflow (`.node-version`) write the lock with npm 11, but harper supports Node 22, which ships npm 10. harper#3006 auto-merged such a lock: the root `utf-8-validate` 6.x didn't satisfy the `^5.0.2` optional peer of the `ws@7` copies in the react-native subtree, so npm 10 wants a nested 5.x where npm 11 doesn't. harper#3015 removed that root declaration. `lockfile-npm-compat.yml` runs `npm ci --strict-peer-deps --dry-run` on Node 22 and Node 24 for every PR and `main` push, with no path filter so it can be a required check. -When the Node 22 leg fails, remove the version conflict at its source. Regenerating the lock with npm 10 doesn't hold: the next npm 11 write prunes the npm-10-only entries again. Release bundling follows production reachability rather than trusting `dev` flags: native-containing trees remain unbundled so the consumer selects its own platform. +Unsatisfied required peers fail this gate rather than being accepted as install warnings. When the Node 22 leg fails, remove the version conflict at its source. Regenerating the lock with npm 10 doesn't hold: the next npm 11 write prunes the npm-10-only entries again. Release bundling follows production reachability rather than trusting `dev` flags: native-containing trees remain unbundled so the consumer selects its own platform. ## A release copies to Docker Hub only the image it booted (`workflows/docker-smoke.yml`, `workflows/publish-docker.yaml`) @@ -30,7 +30,7 @@ Branch protection requires only the `Unit Test` job, which needs every other job Node 26 is the recommended runtime, so every single-version job runs it, and pull requests run unit tests on 24 and 26 (the `lmdb` shard on 26 alone until harper#3349's segfault is fixed) and integration tests on 26 alone. Pushes to `main` run both on 24 and 26. Node 22 runs nightly and on a manual `all` dispatch, as do the native type-strip legs on `22.18.0` and `24.0.0`, the `engines` floors in `package.json`, which are the oldest runtimes that must run the `.ts` sources directly. The nightly schedule runs only on `main`, so a push to a release branch runs that full set: a patch release still ships to Node 22 users. The cherry-pick job dispatches a release branch's unit tests on Node 26 (or 24 when that branch's workflow predates 26) and its integration tests on the Node version whose build artifact its Bun and uWS jobs download (falling back to the unit-test version when all artifact names are matrixed). The integration check names carry the Node version, so moving the PR version means changing the ruleset's required checks in the same release. `.node-version` and `.nvmrc` hold only the major, and every `setup-node` step reading `.node-version` sets `check-latest`, so tooling, release and publish jobs run the newest stable 26.x rather than the runner's cached one or a prerelease. -Integration producers remove `node_modules/harper` before upload and exclude it from artifacts: the globber can still walk an excluded directory link. Each consuming lane's existing npm install recreates the link and bin shims after download. The strict-peer lockfile compatibility gate checks the scoped framework policy without hiding unrelated peer conflicts. +Integration producers remove `node_modules/harper` before upload and exclude it from artifacts: the globber can still walk an excluded directory link. Each consuming lane's existing npm install recreates the link and bin shims after download. ### Unit-test fixtures don't spend the `lmdb` shard on one durable commit per row diff --git a/dependencies.md b/dependencies.md index 81c94923ed..c5911a8bfd 100644 --- a/dependencies.md +++ b/dependencies.md @@ -302,7 +302,7 @@ This is the inverse of the entries below — a dependency we take deliberate ste ## Historical full-text upgrade engines (isolated test fixtures) - Need for usage: Released fulltext 0.5.0, 0.7.0 and 0.8.0 write genuine old checkpoints; the current engine must rebuild incompatible indexes or reuse compatible ones, and prior engines verify safe reopen or downgrade refusal. Source-record IDs and checkpoints are generated during the tests, so static index files do not replace these checks. -- Installation boundary: Harper directly pins the current optional engine; all non-development fulltext entries in the lockfile select 0.9.0. Historical migration aliases are removed from the root dependencies. Separate exact-pinned manifests and integrity-checked lockfiles live under `unitTests/resources/fixtures/fulltext-upgrade/`; the native CI job provisions all three outside the checkout, and existing integration lanes provision only 0.5.0. The previous-engine child exits before the current engine accesses the same index. Production code never imports historical engines or relaxes current ABI validation. +- Installation boundary: Harper directly pins the current optional engine; all non-development fulltext entries in the lockfile select 0.9.0. Root dependencies contain no historical fulltext aliases. Separate exact-pinned manifests and integrity-checked lockfiles live under `unitTests/resources/fixtures/fulltext-upgrade/`; the native CI job provisions all three outside the checkout, and existing integration lanes provision only 0.5.0. The previous-engine child exits before the current engine accesses the same index. Production code never imports historical engines or relaxes current ABI validation. - Local qualification: Run `npm run test:fulltext:install-previous -- /absolute/scratch/fulltext-upgrade`, then set `HARPER_FULLTEXT_UPGRADE_ROOT` to that directory when running the native lifecycle, schema-upgrade, crash-recovery or deployed full-text integration suites. Ordinary resource tests skip historical migration cases when the variable is absent; an explicit but invalid installation fails. CI supplies it; the native unit job also forbids pending tests. - Cost and removal: The dedicated job installs three root packages and their supported platform prebuilds. Integration lanes install only the 0.5.0 fixture; ordinary installs do not provision these migration fixtures. Renovate leaves fixture versions fixed; remove each manifest, lockfile and matching coverage when migration from that version is no longer supported. From b478966830e7b8fb7f35cd242cb755fb4a00e186 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 13:17:06 -0700 Subject: [PATCH 23/26] Reject unknown full-text telemetry configuration keys --- .../config/fullTextTelemetryConfiguration.test.js | 11 +++++++++++ validation/configValidator.ts | 1 + 2 files changed, 12 insertions(+) diff --git a/unitTests/config/fullTextTelemetryConfiguration.test.js b/unitTests/config/fullTextTelemetryConfiguration.test.js index 8f076a301c..83cc67500c 100644 --- a/unitTests/config/fullTextTelemetryConfiguration.test.js +++ b/unitTests/config/fullTextTelemetryConfiguration.test.js @@ -72,6 +72,17 @@ describe('Live full-text telemetry configuration', function () { assert.strictEqual(samples[0].enabled, true); assert.strictEqual(samples[0].histograms, true); }); + it('rejects unknown telemetry keys consistently at boot and live application', () => { + for (const value of [ + { enable: true }, + { enabled: true, histogram: true }, + { intervalMilliseconds: 1000, extra: 1 }, + ]) { + const config = { ...readRootConfig(), analytics: { fullText: value } }; + assert.match(configValidator(config, true).error?.message, /analytics\.fullText/); + assert.throws(() => resolveFullTextTelemetryPolicy(value), /Unknown analytics\.fullText option/); + } + }); it('rejects quoted telemetry values consistently at validation and live application', () => { for (const value of [ { enabled: 'true' }, diff --git a/validation/configValidator.ts b/validation/configValidator.ts index 718ea256fe..32589fba52 100644 --- a/validation/configValidator.ts +++ b/validation/configValidator.ts @@ -449,6 +449,7 @@ export function configValidator(configJson, skipFsValidation = false) { histograms: boolean.optional(), snapshotBytes: boolean.optional(), }) + .unknown(false) .strict() .optional(), }), From 5adea6cf640ac158900914d951ec162ec94dae9f Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 14:00:51 -0700 Subject: [PATCH 24/26] Handle refused analytics metrics and sequence subscription qualification --- .../sourcedfrom-log-identity.test.ts | 39 +++++++++++++++---- resources/analytics/DESIGN.md | 12 ++++-- resources/analytics/write.ts | 10 ++++- .../resources/noAbortAfterTxnLog.test.js | 39 ++++++++++++++++--- 4 files changed, 83 insertions(+), 17 deletions(-) diff --git a/integrationTests/resources/sourcedfrom-log-identity.test.ts b/integrationTests/resources/sourcedfrom-log-identity.test.ts index bcaf53bf15..2b67c54c0f 100644 --- a/integrationTests/resources/sourcedfrom-log-identity.test.ts +++ b/integrationTests/resources/sourcedfrom-log-identity.test.ts @@ -42,6 +42,27 @@ suite('live sourcedFrom local log identities', (ctx: ContextWithHarper) => { return { state: readiness.state, ownerEpoch: readiness.owner_epoch }; } + async function waitForSourceState(condition: (state: any) => boolean, message: string) { + let state; + try { + await waitFor( + async () => { + state = await request('/SourceState/'); + return condition(state); + }, + { timeout: 30_000, message } + ); + } catch (error) { + (error as Error).message += `; last source state: ${JSON.stringify({ + entries: state?.entries?.slice(-20), + records: state?.records, + vector: state?.vector, + })}`; + throw error; + } + return state; + } + async function search(text: string) { const records = await request('/Probe/', 'QUERY', { conditions: [ @@ -145,16 +166,20 @@ suite('live sourcedFrom local log identities', (ctx: ContextWithHarper) => { ]) { await request(`/${source}/${id}`, 'PUT', { id, title, status: 'active', revision: 'r1' }); } + if (kind === 'Generic') { + await waitForSourceState((state) => { + const original = state.records[source].find((record) => record?.id === 's2'); + return state.records[mirror].some( + (record) => record?.id === 's2' && record.value.revision === 'r1' && record.version === original?.version + ); + }, `${mirror} must receive s2 revision r1 before its source is patched`); + } await request(`/${source}/s2`, 'PATCH', { status: 'withdrawn', revision: 'r2' }); const isCaseRecord = ({ id }) => id === 's1' || id === 's2'; - let state; - await waitFor( - async () => { - state = await request('/SourceState/'); - return state.entries.filter((entry) => entry.table === mirror && isCaseRecord(entry)).length >= 3; - }, - { timeout: 30_000, message: `${mirror} must receive all three live writes without a cache read` } + const state = await waitForSourceState( + (state) => state.entries.filter((entry) => entry.table === mirror && isCaseRecord(entry)).length >= 3, + `${mirror} must receive all three live writes without a cache read` ); const sourceEntries = state.entries.filter((entry) => entry.table === source && isCaseRecord(entry)); const mirrorEntries = state.entries.filter((entry) => entry.table === mirror && isCaseRecord(entry)); diff --git a/resources/analytics/DESIGN.md b/resources/analytics/DESIGN.md index 3ec8b338dd..3a92816834 100644 --- a/resources/analytics/DESIGN.md +++ b/resources/analytics/DESIGN.md @@ -32,10 +32,14 @@ Cycles must not overlap. `setInterval` does not await its async callback, so `ru holds a single-flight flag — two cycles reading the same cursor roll the same window up twice and double every count in it. -Still open: `findLastAggregationTime` seeds both markers after a restart from the newest stored -analytics record, whose `time` is an end-of-cycle stamp, so a restart mid-drain can skip the rest of -the backlog; and `storeMetric` discards `table.put()`'s result, so the cursor advances past raw -records whose aggregate rows failed to commit. Both need a cursor persisted with the rows. +`storeMetric` handles synchronous and asynchronous write failures: overload (503) is debug-logged, +other errors are warned, and the sample is dropped without retry. The wedged-commit regression in +`noAbortAfterTxnLog.test.js` keeps rejected metrics from escaping the collector or leaking rejections. + +Still open: restart markers come from the newest stored analytics record, which can skip backlog; +aggregate writes are not joined before advancing the cursor, so failed rows are lost. Synchronous +refusals now drop a row instead of aborting a partially written cycle. Durable replay needs a cursor +persisted with the rows. ## A reset ends the window that was being flushed too (`resources/analytics/write.ts`) diff --git a/resources/analytics/write.ts b/resources/analytics/write.ts index 2d8ed60d67..221c432f93 100644 --- a/resources/analytics/write.ts +++ b/resources/analytics/write.ts @@ -404,7 +404,15 @@ function storeMetric(table: Table, metric: Metric) { ...metric, }; log.trace?.(`storing metric ${JSON.stringify(metricValue)}`); - table.put(metricValue.id, metricValue); + try { + Promise.resolve(table.put(metricValue.id, metricValue)).catch(logMetricWriteError); + } catch (error) { + logMetricWriteError(error); + } +} + +function logMetricWriteError(error) { + log[error?.statusCode === 503 ? 'debug' : 'warn']?.('Unable to save analytics metric', error); } interface ResourceUsage extends Partial { diff --git a/unitTests/resources/noAbortAfterTxnLog.test.js b/unitTests/resources/noAbortAfterTxnLog.test.js index 97b4b8f281..a8143c01c5 100644 --- a/unitTests/resources/noAbortAfterTxnLog.test.js +++ b/unitTests/resources/noAbortAfterTxnLog.test.js @@ -1,7 +1,7 @@ require('../testUtils'); const assert = require('node:assert'); const { setTimeout: delay } = require('node:timers/promises'); -const { setupTestDBPath } = require('../testUtils'); +const { setupTestDBPath, ensureSystemTables } = require('../testUtils'); const { waitFor } = require('../waitFor'); const { table } = require('#src/resources/databases'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); @@ -23,6 +23,7 @@ const { publishOriginFloor, reservedLocalKey } = require('#src/resources/originC const { replayLogs } = require('#src/resources/replayLogs'); const { setAbortMarkersForTests } = require('#src/resources/transactionLogAbortMarkers'); const harperLogger = require('#src/utility/logging/harper_logger'); +const { storeRocksDBStatsMetrics } = require('#src/resources/analytics/write'); const { Transaction, constants } = require('@harperfast/rocksdb-js'); const RETRY_NOW_VALUE = constants.RETRY_NOW_VALUE; require('#src/server/serverHelpers/serverUtilities'); @@ -1414,6 +1415,20 @@ describe('Transactions are never aborted once durably written to the transaction it('releases a deferred record lock via the hold-deadline timer when a native commit wedges and never settles', async function () { this.timeout(20000); + await ensureSystemTables(); + const analyticsTable = table({ + table: 'hdb_analytics', + database: 'system', + audit: true, + trackDeletes: false, + attributes: ['id', 'metric', 'path', 'method', 'type'].map((name) => ({ + name, + ...(name === 'id' ? { isPrimaryKey: true } : {}), + })), + }); + const metricDatabases = { testNoAbortA: { NoAbortAuditedA: AuditedA } }; + storeRocksDBStatsMetrics(analyticsTable, metricDatabases, Date.now(), undefined); + await new Promise(setImmediate); const recordId = id('lock-wedged-commit'); await AuditedA.put({ id: recordId, writer: 'seed', count: 0 }); @@ -1459,10 +1474,24 @@ describe('Transactions are never aborted once durably written to the transaction ); const contender = await AuditedA.lock(recordId, { lease: 1000, timeout: 2500 }); await contender.unlock(); - - releaseWedged(); - await savePromise; - await holder.unlock(); + try { + const metricId = [Date.now(), 123]; + await assert.rejects( + async () => analyticsTable.put(metricId, { id: metricId, metric: 'overload-probe' }), + (error) => error.statusCode === 503, + 'the wedged native commit must activate real global write shedding' + ); + assert.doesNotThrow( + () => storeRocksDBStatsMetrics(analyticsTable, metricDatabases, Date.now(), 1000), + 'refused background metrics must not escape the aggregation callback' + ); + await new Promise(setImmediate); + assert.deepStrictEqual(unhandled, [], 'refused background metric writes must not leak unhandled rejections'); + } finally { + releaseWedged(); + await savePromise; + await holder.unlock(); + } }); it('does not re-stage an in-flight sibling write onto a fresh write when a shared ImmediateTransaction had an earlier retry', async function () { From 9f3a91f3e6bb96f0ee18bd9b745981a9d7912845 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 14:38:00 -0700 Subject: [PATCH 25/26] Reconcile recording after rejected closes and finish benchmark children --- benchmarks/fulltext-analytics.mjs | 9 +- .../sourcedfrom-log-identity.test.ts | 11 ++ resources/analytics/DESIGN.md | 2 +- resources/analytics/fullTextTelemetry.ts | 1 + .../nativeFullTextTelemetryClose.test.js | 149 ++++++++++++++++++ .../nativeFullTextTelemetryShutdown.test.js | 36 ++--- 6 files changed, 183 insertions(+), 25 deletions(-) diff --git a/benchmarks/fulltext-analytics.mjs b/benchmarks/fulltext-analytics.mjs index 9715dcfdf7..71307c8865 100644 --- a/benchmarks/fulltext-analytics.mjs +++ b/benchmarks/fulltext-analytics.mjs @@ -124,14 +124,17 @@ async function spawnMocha(profile, pair) { child.stderr.on('data', capture); let timedOut = false; let killTimer; + let spawnError; const timer = setTimeout(() => { timedOut = true; child.kill('SIGTERM'); killTimer = setTimeout(() => child.kill('SIGKILL'), 5000).unref(); }, options.duration + 120_000); const result = await new Promise((resolveChild) => { - child.once('error', (error) => resolveChild({ code: null, signal: null, spawnError: error.message })); - child.once('close', (code, signal) => resolveChild({ code, signal })); + child.on('error', (error) => { + spawnError ??= error.message; + }); + child.once('close', (code, signal) => resolveChild({ code, signal, spawnError })); }).finally(() => { clearTimeout(timer); clearTimeout(killTimer); @@ -145,7 +148,7 @@ async function spawnMocha(profile, pair) { 2 ) ); - if (result.code !== 0 || timedOut || logError || !existsSync(join(directory, 'result.json'))) + if (result.code !== 0 || spawnError || timedOut || logError || !existsSync(join(directory, 'result.json'))) throw new Error(`${profile} pair ${pair} failed; retained ${directory}`); return JSON.parse(readFileSync(join(directory, 'result.json'), 'utf8')); } diff --git a/integrationTests/resources/sourcedfrom-log-identity.test.ts b/integrationTests/resources/sourcedfrom-log-identity.test.ts index 2b67c54c0f..4bddc3ad3e 100644 --- a/integrationTests/resources/sourcedfrom-log-identity.test.ts +++ b/integrationTests/resources/sourcedfrom-log-identity.test.ts @@ -213,6 +213,17 @@ suite('live sourcedFrom local log identities', (ctx: ContextWithHarper) => { 6, `separate local commits must have distinct keys: ${JSON.stringify(entries)}` ); + if (kind === 'Generic') { + await request(`/${source}/s2`, 'PUT', { id: 's2', title: 'burst', revision: 'r3' }); + await request(`/${source}/s2`, 'PATCH', { revision: 'r4' }); + await waitForSourceState((state) => { + const original = state.records[source].find((record) => record?.id === 's2'); + return state.records[mirror].some( + (record) => record?.id === 's2' && record.value.revision === 'r4' && record.version === original?.version + ); + }, `${mirror} must receive the newest burst revision and source version`); + await assertIndexProgress('genericburst', [0, 0.5, 0.25], before); + } }); test(`${kind} multi-record live apply keeps each transaction grouped under one fresh key`, async () => { diff --git a/resources/analytics/DESIGN.md b/resources/analytics/DESIGN.md index 3a92816834..a235951980 100644 --- a/resources/analytics/DESIGN.md +++ b/resources/analytics/DESIGN.md @@ -79,7 +79,7 @@ always a caller's measure. `fullTextTelemetry.ts` keeps private per-handle baselines and carries exact counters into a stable worker/index/role series. `nativeFullTextTelemetry.test.js` pins reader churn, wrap/precision, optional recording unions and peer-gauge aggregation; cached aggregate names include null leaves, so a live peer's queue age survives a newest null observation. `nativeFullTextTelemetryClose.test.js` holds backend close across live-peer timer ticks and recording changes: closing handles are neither sampled nor reconfigured until settlement. Completed registrations release references; inactive groups remain until their final row drains. Pending export has a 128-row bound with explicit drop counts. Graceful HTTP/job exit reuses the serialized analytics flush after owner cleanup, drains pending rows and samples open handles without overflowing that bound (`nativeFullTextTelemetryShutdown.test.js`). -`fullTextTelemetryRuntime.ts` joins removable internal consumer demand with live operator policy. The consumer hook is an internal diagnostic/test seam with no production caller yet. Only operator demand enables persistence; persisted optional fields reflect the union of recording requests. No demand means no timer; no cold handles open. Exact counters are decimal strings; `cumulative` retains the newest worker row instead of averaging. Lost final counts force a new series on the next live observation. Overflow prefers replaceable live rows, ordered by last refresh, before final rows; dropped live totals remain recoverable. Main also filters worker rows by its receive-time policy; transition losses, including final rows, do not increment `droppedRows`. Failed writer closes resume collection while the engine remains installed. `aggregationCycle.test.js` verifies counters above 2^53 survive storage and `get_analytics`. +`fullTextTelemetryRuntime.ts` joins removable internal consumer demand with live operator policy. The consumer hook is an internal diagnostic/test seam with no production caller yet. Only operator demand enables persistence; persisted optional fields reflect the union of recording requests. No demand means no timer; no cold handles open. Exact counters are decimal strings; `cumulative` retains the newest worker row instead of averaging. Lost final counts force a new series on the next live observation. Overflow prefers replaceable live rows, ordered by last refresh, before final rows; dropped live totals remain recoverable. Main also filters worker rows by its receive-time policy; transition losses, including final rows, do not increment `droppedRows`. Failed closes reconcile deferred recording even with no demand; a never-enabled handle may receive an idempotent configure-off call. `aggregationCycle.test.js` verifies counters above 2^53 survive storage and `get_analytics`. `watchFullTextTelemetryConfiguration` initializes and reloads through the shared watcher's nested root, environment overlay and transactional `refreshConfigSubtree`; it never reads the missing camel-case flat aggregate. Invalid edits retain valid policy. Missing files retain valid policy and overrides, matching the root watcher; valid edits retire only changed overrides. `fullTextTelemetryConfiguration.test.js` pins initialization, override retirement and deletion/recreation. Native runtime/query policy remains frozen separately. Binding telemetry types are structural: `buildDist.test.mjs` checks compilation and emitted declarations without the optional package. diff --git a/resources/analytics/fullTextTelemetry.ts b/resources/analytics/fullTextTelemetry.ts index 3114cdccf8..a9924095e9 100644 --- a/resources/analytics/fullTextTelemetry.ts +++ b/resources/analytics/fullTextTelemetry.ts @@ -271,6 +271,7 @@ export class FullTextTelemetryCollector { entry.closing = false; this.#closeHook(() => { if (this.collecting) this.#observe(entry); + else if (!entry.configured) this.#configure(entry); this.#schedule(); }); } diff --git a/unitTests/resources/nativeFullTextTelemetryClose.test.js b/unitTests/resources/nativeFullTextTelemetryClose.test.js index 7cb889ed25..93d9c1e2e5 100644 --- a/unitTests/resources/nativeFullTextTelemetryClose.test.js +++ b/unitTests/resources/nativeFullTextTelemetryClose.test.js @@ -11,6 +11,7 @@ const { encodeFullTextCursorPayload, } = require('#src/resources/indexes/fullTextDerivedIndex'); const { DERIVED_INDEX_ACCEPTED } = require('#src/resources/derivedIndexRuntime'); +const { FullTextTelemetryCollector } = require('#src/resources/analytics/fullTextTelemetry'); const { subscribeFullTextTelemetry, registerNativeFullTextTelemetry, @@ -26,6 +27,154 @@ const limits = { maxBatchBytes: 8 * 1024 * 1024, }; +for (const demand of ['consumer', 'operator', 'default']) { + const description = + demand === 'default' + ? 'keeps default recording disabled after a rejected close' + : `disables native recording after a rejected close when the last ${demand} demand was removed`; + it(description, async function () { + this.timeout(20_000); + if (!(await supportsPublishedFullTextBinding())) this.skip(); + const binding = await loadFullTextNativeBinding(); + const id = `close-disabled-${demand}-${process.pid}`; + const writer = await binding.openNativeFullTextIndex({ + path: join(setupTestDBPath(), id), + indexId: id, + generation: 'g1', + fields: [{ name: 'title' }], + analyzer: 'english@2', + positions: true, + limits, + }); + let errors = 0; + const collector = new FullTextTelemetryCollector({ onError: () => errors++ }); + const policy = { histograms: true, snapshotBytes: true, intervalMilliseconds: 1000 }; + let removeDemand = () => {}; + if (demand === 'consumer') removeDemand = collector.subscribe(() => {}, policy); + else if (demand === 'operator') { + collector.setOperatorPolicy({ ...policy, enabled: true }); + removeDemand = () => collector.setOperatorPolicy({ enabled: false }); + } + const recording = () => { + const { histogramsEnabled, snapshotBytesEnabled } = writer.telemetry().native; + return { histogramsEnabled, snapshotBytesEnabled }; + }; + const initiallyRecording = demand !== 'default'; + try { + collector.register(writer, id, 'writer'); + assert.deepStrictEqual(recording(), { + histogramsEnabled: initiallyRecording, + snapshotBytesEnabled: initiallyRecording, + }); + await writer.applyMutationBatch({ upserts: [{ id: 'dirty', fields: { title: 'catalog widget' } }], deletes: [] }); + collector.beforeClose(writer); + const closing = writer.close({ mode: 'require-clean' }); + removeDemand(); + assert.strictEqual(collector.collecting, false); + assert.strictEqual(collector.scheduled, false); + assert.deepStrictEqual(recording(), { + histogramsEnabled: initiallyRecording, + snapshotBytesEnabled: initiallyRecording, + }); + await assert.rejects(closing, (error) => error.code === 'E_DIRTY_CLOSE'); + collector.closeFailed(writer); + assert.strictEqual(writer.status().state, 'open'); + assert.strictEqual(collector.collecting, false); + assert.strictEqual(collector.scheduled, false); + assert.deepStrictEqual(recording(), { histogramsEnabled: false, snapshotBytesEnabled: false }); + await writer.applyMutationBatch({ upserts: [{ id: 'later', fields: { title: 'still writable' } }], deletes: [] }); + assert.strictEqual(writer.status().state, 'open'); + assert.deepStrictEqual(recording(), { histogramsEnabled: false, snapshotBytesEnabled: false }); + assert.strictEqual(errors, 0); + } finally { + removeDemand(); + collector.beforeClose(writer); + try { + await writer.close({ mode: 'rollback' }); + collector.afterClose(writer); + } finally { + collector.stop(); + } + } + }); +} + +it('reconciles removed recording demand through a real backend close failure', async function () { + this.timeout(20_000); + if (!(await supportsPublishedFullTextBinding())) this.skip(); + const binding = await loadFullTextNativeBinding(); + const id = `close-backend-disabled-${process.pid}`; + const lifecycle = new NativeFullTextDerivedIndexLifecycle({ + storePath: join(setupTestDBPath(), id), + storeName: 'catalog.Product.title', + indexId: id, + sourceGeneration: 'generation-1', + fields: [{ name: 'title' }], + analyzer: 'english@2', + positions: true, + limits, + binding, + }); + await lifecycle.initialize({ reclaimRetired: false }); + let writer; + let removeConsumer = () => {}; + removeConsumer = subscribeFullTextTelemetry( + (sample) => { + if (sample.index === id && sample.stage === 'closing') removeConsumer(); + }, + { histograms: true, snapshotBytes: true, intervalMilliseconds: 1000 } + ); + const backend = new FullTextDerivedIndexBackend({ + id, + lifecycle: { + inspect: () => lifecycle.inspect(), + reset: (epoch) => lifecycle.reset(epoch), + async open() { + writer = await lifecycle.open(); + return writer; + }, + }, + closeTimeoutMilliseconds: 5000, + shutdownTimeoutMilliseconds: 10_000, + }); + backend.attach({ + isOwnerEpoch: (epoch) => epoch === 1n, + getReadiness: () => ({ state: 'ready', ownerEpoch: 1n, rebuildAttempts: 0 }), + publicationChanged() {}, + }); + const cursor = { format: 1, logs: { local: 1 } }; + try { + backend.getDurableCursor(); + assert.strictEqual( + backend.deliver({ ownerEpoch: 1n, transactions: [], records: [], through: cursor, bytes: 0 }), + DERIVED_INDEX_ACCEPTED + ); + backend.flush(); + await waitFor(() => backend.getDurableCursor()?.logs.local === 1, 5000); + assert.strictEqual(writer.telemetry().native.histogramsEnabled, true); + assert.strictEqual(writer.telemetry().native.snapshotBytesEnabled, true); + await writer.applyMutationBatch({ upserts: [{ id: 'dirty', fields: { title: 'catalog widget' } }], deletes: [] }); + await assert.rejects(backend.shutdown(1n), /did not prove quiescence/); + assert.strictEqual(writer.telemetry().native.operations.close.errors.E_DIRTY_CLOSE, 1n); + assert.strictEqual(writer.status().state, 'open'); + assert.strictEqual(fullTextTelemetryCollector.collecting, false); + assert.strictEqual(fullTextTelemetryCollector.scheduled, false); + const { histogramsEnabled, snapshotBytesEnabled } = writer.telemetry().native; + assert.deepStrictEqual( + { histogramsEnabled, snapshotBytesEnabled }, + { histogramsEnabled: false, snapshotBytesEnabled: false } + ); + await writer.applyMutationBatch({ upserts: [{ id: 'later', fields: { title: 'still writable' } }], deletes: [] }); + await writer.publish(encodeFullTextCursorPayload(cursor)); + await backend.shutdown(1n); + assert.strictEqual(writer.status().state, 'closed'); + } finally { + removeConsumer(); + if (writer && writer.status().state !== 'closed') await writer.close({ mode: 'rollback' }); + await backend.shutdown(1n); + } +}); + it('keeps observing an installed real writer after a rejected require-clean close and closes on retry', async function () { this.timeout(20_000); if (!(await supportsPublishedFullTextBinding())) this.skip(); diff --git a/unitTests/resources/nativeFullTextTelemetryShutdown.test.js b/unitTests/resources/nativeFullTextTelemetryShutdown.test.js index c86418fa03..c3f8be56e4 100644 --- a/unitTests/resources/nativeFullTextTelemetryShutdown.test.js +++ b/unitTests/resources/nativeFullTextTelemetryShutdown.test.js @@ -22,6 +22,17 @@ const { randomUUID } = require('node:crypto'); const { jobWorkerOptions } = require('#src/server/jobs/jobRunner'); const { JOB_STATUS_ENUM, SYSTEM_TABLE_NAMES } = require('#src/utility/hdbTerms'); +function persistedClosedMetric(workerId, label) { + for (const { value } of getDatabases().system.hdb_raw_analytics.primaryStore.getRange({ + start: false, + end: Infinity, + })) { + if (value.threadId !== workerId) continue; + const row = value.metrics.find((row) => row.path === label && row.native_state === 'closed'); + if (row) return row; + } +} + it('commits main-thread final analytics before exit and reopens the real store', async function () { this.timeout(60_000); const { execFile } = require('node:child_process'); @@ -167,17 +178,9 @@ it('flushes actual job-worker analytics before cleanup acknowledgement and persi assert.ok(ackIndex > finalIndex, 'terminal analytics must finish before job cleanup acknowledgement'); const finalReportIndex = messages.findLastIndex((message) => message.type === 'analytics-report'); assert.ok(finalReportIndex > finalIndex && finalReportIndex < ackIndex); - const stored = []; - for (const { value } of getDatabases().system.hdb_raw_analytics.primaryStore.getRange({ - start: false, - end: Infinity, - })) - if (value.threadId === workerId) stored.push(...value.metrics); - assert.ok( - stored.some( - (row) => row.path === label && row.native_state === 'closed' && row.native_operations_close_completed === '1' - ) - ); + const persisted = () => persistedClosedMetric(workerId, label); + await waitFor(persisted, 5000); + assert.strictEqual(persisted().native_operations_close_completed, '1'); } finally { try { if (worker && worker.threadId !== -1) await worker.terminate(); @@ -258,16 +261,7 @@ it('delivers final native telemetry after deferred scope close and before worker .flatMap((message) => message.report?.metrics ?? []) .find((row) => row.path === label + '-previous'); assert.strictEqual(pending?.native_state, 'closed', 'shutdown must preserve an already pending closed-handle row'); - const persisted = () => { - for (const { value } of getDatabases().system.hdb_raw_analytics.primaryStore.getRange({ - start: false, - end: Infinity, - })) { - if (value.threadId !== workerId) continue; - const row = value.metrics.find((row) => row.path === label && row.native_state === 'closed'); - if (row) return row; - } - }; + const persisted = () => persistedClosedMetric(workerId, label); await waitFor(persisted, 5000); assert.strictEqual(persisted().native_operations_close_completed, '1'); } finally { From 7b64407a8995e13aeff7972fe7d31889585a33f3 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Wed, 4 Nov 2026 14:56:17 -0700 Subject: [PATCH 26/26] Observe live source reads in subscription qualification --- .../sourcedfrom-log-identity.test.ts | 24 +++++++++++++++++++ .../sourcedfrom-log-identity/resources.ts | 4 +++- resources/analytics/fullTextTelemetry.ts | 2 +- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/integrationTests/resources/sourcedfrom-log-identity.test.ts b/integrationTests/resources/sourcedfrom-log-identity.test.ts index 4bddc3ad3e..1996eafe8c 100644 --- a/integrationTests/resources/sourcedfrom-log-identity.test.ts +++ b/integrationTests/resources/sourcedfrom-log-identity.test.ts @@ -57,6 +57,7 @@ suite('live sourcedFrom local log identities', (ctx: ContextWithHarper) => { entries: state?.entries?.slice(-20), records: state?.records, vector: state?.vector, + genericSourceReads: state?.genericSourceReads, })}`; throw error; } @@ -160,6 +161,7 @@ suite('live sourcedFrom local log identities', (ctx: ContextWithHarper) => { const source = `${kind}Source`; const mirror = `${kind}Mirror`; const before = await indexEpochs(); + const sourceReads = kind === 'Generic' ? (await request('/SourceState/')).genericSourceReads : undefined; for (const [id, title] of [ ['s1', 'alpha'], ['s2', 'beta'], @@ -223,11 +225,17 @@ suite('live sourcedFrom local log identities', (ctx: ContextWithHarper) => { ); }, `${mirror} must receive the newest burst revision and source version`); await assertIndexProgress('genericburst', [0, 0.5, 0.25], before); + assert.strictEqual( + (await request('/SourceState/')).genericSourceReads, + sourceReads, + 'live apply must not re-fetch the source' + ); } }); test(`${kind} multi-record live apply keeps each transaction grouped under one fresh key`, async () => { const before = await indexEpochs(); + const sourceReads = kind === 'Generic' ? (await request('/SourceState/')).genericSourceReads : undefined; await request(`/${kind}Source/`, 'PUT', [ { id: 'batch1', title: 'one', revision: 'r1' }, { id: 'batch2', title: 'two', revision: 'r1' }, @@ -263,6 +271,22 @@ suite('live sourcedFrom local log identities', (ctx: ContextWithHarper) => { source.map(({ version }) => version) ); await assertIndexProgress(`${kind.toLowerCase()}batch`, kind === 'Direct' ? [0, 1, 1] : [0, -1, 1], before); + if (kind === 'Generic') { + assert.strictEqual( + (await request('/SourceState/')).genericSourceReads, + sourceReads, + 'live transaction apply must not re-fetch the source' + ); + const response = await fetch(`${ctx.harper.httpURL}/GenericMirror/source-read-observation`, { + headers, + signal: AbortSignal.timeout(45_000), + }); + assert.strictEqual(response.status, 404, await response.text()); + assert.ok( + (await request('/SourceState/')).genericSourceReads > sourceReads, + 'a real cache miss must exercise the observed source get' + ); + } }); } }); diff --git a/integrationTests/resources/sourcedfrom-log-identity/resources.ts b/integrationTests/resources/sourcedfrom-log-identity/resources.ts index f499f2fcd5..19408e63ab 100644 --- a/integrationTests/resources/sourcedfrom-log-identity/resources.ts +++ b/integrationTests/resources/sourcedfrom-log-identity/resources.ts @@ -2,11 +2,13 @@ declare const tables: any; declare const Resource: any; const { DirectSource, DirectMirror, GenericSource, GenericMirror, Probe } = tables; +let genericSourceReads = 0; DirectMirror.sourcedFrom(DirectSource); GenericMirror.sourcedFrom({ name: 'ForwardedTableSource', async get(id) { + genericSourceReads++; const record = await GenericSource.get(id, {}); return record ? { ...record } : undefined; }, @@ -50,6 +52,6 @@ export class SourceState extends Resource { ]) ); const readiness = Probe.indices.vector.customIndex.derivedHost.readiness(); - return { entries, records, vector: { ...readiness, ownerEpoch: String(readiness.ownerEpoch) } }; + return { entries, records, vector: { ...readiness, ownerEpoch: String(readiness.ownerEpoch) }, genericSourceReads }; } } diff --git a/resources/analytics/fullTextTelemetry.ts b/resources/analytics/fullTextTelemetry.ts index a9924095e9..c219ce43b3 100644 --- a/resources/analytics/fullTextTelemetry.ts +++ b/resources/analytics/fullTextTelemetry.ts @@ -3,7 +3,7 @@ import { METRIC } from './metadata.ts'; import type { NativeFullTextTelemetryHandle } from '../indexes/fullTextNativeBinding.ts'; type ConfigureTelemetryOptions = Parameters[0]; -export type FullTextTelemetrySnapshot = ReturnType; +type FullTextTelemetrySnapshot = ReturnType; export type FullTextTelemetryPolicy = Required & { enabled: boolean;