From 0863ef8c52edf99e206ff35c4eb5297dd04ccfe1 Mon Sep 17 00:00:00 2001 From: Walter Pearce Date: Thu, 29 Oct 2026 11:47:26 -0700 Subject: [PATCH] Add a painter-jobs service that precomputes the dashboard totals into Stats nodes and hosts the Map rebuild, so render only reads. --- docker-compose.yaml | 20 +++ docker/queries.md | 2 + render/.env.example | 9 ++ render/Dockerfile | 3 +- render/README.md | 46 +++++- render/package.json | 1 + render/server/config.mjs | 13 ++ render/server/db.mjs | 59 +++++++ render/server/index.mjs | 123 ++++++++------ render/server/jobs/index.mjs | 196 +++++++++++++++++++++++ render/server/stats.mjs | 68 ++++++++ render/src/api.ts | 22 ++- render/src/components/BuildView.tsx | 1 + render/src/components/DiffsView.tsx | 1 + render/src/components/NativeView.tsx | 1 + render/src/components/PublishersView.tsx | 3 +- render/src/components/UnsafeView.tsx | 2 + render/src/components/meta.tsx | 27 +++- src/migrate.rs | 8 + 19 files changed, 545 insertions(+), 60 deletions(-) create mode 100644 render/server/db.mjs create mode 100644 render/server/jobs/index.mjs create mode 100644 render/server/stats.mjs diff --git a/docker-compose.yaml b/docker-compose.yaml index 82c6443..9a27b85 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -44,6 +44,26 @@ services: - NEO4J_PASSWORD=changeme123 - NEO4J_DATABASE=neo4j - RENDER_ALLOW_CYPHER=true + - RENDER_JOBS_URL=http://jobs:8080 + depends_on: + - neo4j + + # Background jobs (render/server/jobs/index.mjs, same image as render): refreshes the dashboard + # totals (Stats nodes) every JOBS_STATS_INTERVAL_MS and runs Map rebuilds, so render only reads. + # Control API on 127.0.0.1:8081 for scripts: POST /api/stats/refresh, /api/lod/rebuild, GET /api/status. + jobs: + build: ./render + command: ["node", "server/jobs/index.mjs"] + restart: unless-stopped + ports: + - 127.0.0.1:8081:8080 + environment: + - NEO4J_URI=bolt://neo4j:7687 + - NEO4J_USER=neo4j + - NEO4J_PASSWORD=changeme123 + - NEO4J_DATABASE=neo4j + - JOBS_STATS_INTERVAL_MS=900000 + - JOBS_LOD_UPDATE_INTERVAL_MS=0 depends_on: - neo4j diff --git a/docker/queries.md b/docker/queries.md index 31bf2ba..1d8ca58 100644 --- a/docker/queries.md +++ b/docker/queries.md @@ -35,6 +35,8 @@ CREATE INDEX idx_crate_downloads IF NOT EXISTS FOR (c:Crate) ON (c.downloads); - `const_user_id` ... `idx_crate_downloads`: created by the `registry` job itself (`db::REGISTRY_SCHEMA_CYPHER`). `idx_published_at` is a relationship range index. - The Map build adds its own `Crate.lod_*` indexes and `LOD_DEP` (see `render/README.md`). +- The render jobs service creates `const_stats_id` on `(:Stats {id})`, the dashboard totals it + precomputes (JSON `payload`, `computed_at`); `MATCH (s:Stats) DETACH DELETE s` is always safe. Enterprise only: diff --git a/render/.env.example b/render/.env.example index 696d8b7..8fceadd 100644 --- a/render/.env.example +++ b/render/.env.example @@ -22,3 +22,12 @@ RENDER_LOD_WORKER_HEAP_MB=8192 RENDER_LOD_BUDGET=600 RENDER_LOD_MAX_LEVEL=12 RENDER_LOD_CHECKPOINT=/tmp/painter-lod-checkpoint.json + +# Background jobs service (server/jobs/index.mjs, see README "Background jobs"). +# render: forward stats refreshes and Map rebuilds there instead of running them itself. +#RENDER_JOBS_URL=http://127.0.0.1:8081 +# jobs: dashboard totals (Stats nodes) refresh interval (0 = on request only), per-payload query timeout, +# and an optional timer for the cheap Map update (0 = off). +JOBS_STATS_INTERVAL_MS=900000 +JOBS_QUERY_TIMEOUT_MS=600000 +JOBS_LOD_UPDATE_INTERVAL_MS=0 diff --git a/render/Dockerfile b/render/Dockerfile index 6a3bb11..25c298e 100644 --- a/render/Dockerfile +++ b/render/Dockerfile @@ -19,6 +19,7 @@ COPY server ./server COPY --from=build /app/dist ./dist EXPOSE 8080 USER node +# Shell form so $PORT expands: the jobs service (CMD node server/jobs/index.mjs) answers the same path. HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ - CMD wget -qO- http://127.0.0.1:8080/api/health > /dev/null || exit 1 + CMD wget -qO- "http://127.0.0.1:${PORT}/api/health" > /dev/null || exit 1 CMD ["node", "server/index.mjs"] diff --git a/render/README.md b/render/README.md index 46a179e..889984e 100644 --- a/render/README.md +++ b/render/README.md @@ -32,7 +32,45 @@ Settings are env vars read once at startup; defaults match the compose neo4j. Se container to a host Neo4j use `bolt://host.docker.internal:7687` plus `--add-host=host.docker.internal:host-gateway`), `NEO4J_MAX_ROWS` (row cap on every query, Cypher panel included), `RENDER_ALLOW_CYPHER=false` (disables the Cypher panel and `POST /api/cypher`), -`RENDER_LOD_WORKER_HEAP_MB` (heap for the in-server Map build). +`RENDER_LOD_WORKER_HEAP_MB` (heap for the in-server Map build), `RENDER_JOBS_URL` (hand background +work to the jobs service below). + +## Background jobs + +Every dashboard's overview tiles (unsafe, build scripts, native code, diffs, registry, publisher +leaderboards, and the status bar counts) aggregate the whole graph: label scans of 3.2M versions, +whole relationship types, seconds each and tens of seconds under import load. Rather than paying for +that on a request, `server/jobs/index.mjs` runs as its own service (`jobs` in `docker-compose.yaml`, +same image, `command: node server/jobs/index.mjs`) and is the only writer of derived data: + +- every `JOBS_STATS_INTERVAL_MS` (15 min; 0 = on request only) it recomputes each payload in + `server/stats.mjs` (`STATS_JOBS`: `overview`, `unsafe:{latest,all}`, `build:*`, `native:*`, + `diffs:*`, `registry`, `leaderboards`), one scan at a time, and stores it as JSON on + `(:Stats {id, payload, computed_at, elapsed_ms, bytes})` (constraint `const_stats_id`, created at + start). A timer run skips ids another instance refreshed within half the interval. +- the Map build/update worker (below) runs here when asked, and `JOBS_LOD_UPDATE_INTERVAL_MS` can + run the cheap update on a timer. + +Control API on `$PORT` (compose publishes it on `127.0.0.1:8081`): `GET /api/health`, +`GET /api/status` (both runs plus the stored `computed_at` per id), `POST /api/stats/refresh` +(`{ids?: [...]}`, 202, 409 while one runs), `GET /api/lod/status`, `POST /api/lod/rebuild`, +`POST /api/lod/update`. Trigger a refresh when a painter stage finishes instead of waiting for the +timer: + +``` +curl -X POST http://127.0.0.1:8081/api/stats/refresh +curl -X POST http://127.0.0.1:8081/api/stats/refresh -H 'content-type: application/json' -d '{"ids":["registry","leaderboards"]}' +``` + +The render routes read the `Stats` node (one index seek) and report `computedAt` / `source: +"precomputed"`; the views print "totals computed N min ago". With no node yet (fresh database, no +jobs service) a route falls back to the live aggregate with its old 60 s / 10 min cache and +`source: "live"`. With `RENDER_JOBS_URL` set, render forwards `POST /api/stats/refresh` and +`POST /api/lod/{rebuild,update}` to the jobs service and shows its job status under +`/api/lod/status`, so the graph has one writer; without it, render still runs the Map worker itself +as before. `GET /api/stats/status` lists stored ages and the jobs service's view. + +Locally: `npm run start:jobs` (port 8081) next to `RENDER_JOBS_URL=http://127.0.0.1:8081 npm run dev`. ## Indexes @@ -121,7 +159,8 @@ interval. To keep it from starving the host: `NODE_OPTIONS=--max-old-space-size=6144 nice -n 19 ionice -c 3 npm run lod:build -- --reuse-edges --resume`. The UI (Rebuild/Update) and `POST /api/lod/rebuild` / `POST /api/lod/update` run the same jobs in a -worker thread, one at a time (409 if busy), heap-capped by `RENDER_LOD_WORKER_HEAP_MB`. The server +worker thread, one at a time (409 if busy), heap-capped by `RENDER_LOD_WORKER_HEAP_MB`: in the jobs +service when `RENDER_JOBS_URL` is set (render forwards the request), else inside render. The server reloads its viewport index when a job finishes or it sees a new `built_at`. `GET /api/lod/viewport` picks the finest grid level whose cells in the bbox fit the budget and are at @@ -143,7 +182,8 @@ obvious ones: | `GET /api/graph/deps/chain` | Deps view; `meta.roots/depth/truncated` | | `GET /api/lod/*` | Map status, viewport, locate, search; viewport is 503 until a build is loaded | | `GET /api/callers?crate&fn&match&version&depth&limit` | Callers rows; `match=exact\|contains` | -| `GET /api/{unsafe,build,native,diffs}/{stats,top,crate/:name}` | leaderboard tiles, sorted/filtered rows (`hasMore` for paging), one crate's per-version history; stats cached 60 s | +| `GET /api/{unsafe,build,native,diffs}/{stats,top,crate/:name}` | leaderboard tiles, sorted/filtered rows (`hasMore` for paging), one crate's per-version history; stats come from the jobs service's `Stats` nodes (`computedAt`), live + cached 60 s without one | +| `GET /api/stats/status`, `POST /api/stats/refresh` | stored `Stats` ages; refresh forwarded to `RENDER_JOBS_URL` (503 without one) | | `GET /api/registry/*`, `/api/publishers/*`, `/api/releases*`, `/api/supply/*` | registry, publisher, release, and supply-chain data | | `POST /api/cypher` | `{query, params}`, read-only, row-capped; 403 when disabled | diff --git a/render/package.json b/render/package.json index 3b78c0d..0635a2b 100644 --- a/render/package.json +++ b/render/package.json @@ -10,6 +10,7 @@ "dev:web": "vite", "build": "tsc --noEmit && vite build", "start": "node server/index.mjs", + "start:jobs": "PORT=8081 node server/jobs/index.mjs", "typecheck": "tsc --noEmit", "lod:build": "node server/lod/cli.mjs build", "lod:update": "node server/lod/cli.mjs update", diff --git a/render/server/config.mjs b/render/server/config.mjs index 941ec8f..6bd9ada 100644 --- a/render/server/config.mjs +++ b/render/server/config.mjs @@ -39,6 +39,17 @@ export const config = { // Positions every N iterations, so a crash can be resumed. checkpointPath: env('RENDER_LOD_CHECKPOINT', '/tmp/painter-lod-checkpoint.json'), }, + // Base URL of the jobs service (server/jobs/index.mjs). When set, render never runs a LOD worker or a stats + // refresh itself: POST /api/lod/* and /api/stats/refresh are forwarded there, so the graph has one writer. + jobsUrl: env('RENDER_JOBS_URL', '').replace(/\/+$/, ''), + jobs: { + // Dashboard totals (Stats nodes) are recomputed this often; 0 disables the timer (manual refresh only). + statsIntervalMs: int('JOBS_STATS_INTERVAL_MS', 15 * 60_000), + // Per-payload limit: unsafe:all is ~95 s on a cold local database and slower under import load. + queryTimeoutMs: int('JOBS_QUERY_TIMEOUT_MS', 600_000), + // The cheap Map update (place crates imported since the last build); 0 = only on request. + lodUpdateIntervalMs: int('JOBS_LOD_UPDATE_INTERVAL_MS', 0), + }, }; export const describeConfig = () => ({ @@ -55,4 +66,6 @@ export const describeConfig = () => ({ depsMaxEdges: config.depsMaxEdges, depsDefaultEdges: config.depsDefaultEdges, lod: { iterations: config.lod.iterations, budget: config.lod.budget, maxLevel: config.lod.maxLevel }, + jobsUrl: config.jobsUrl || null, + jobs: config.jobs, }); diff --git a/render/server/db.mjs b/render/server/db.mjs new file mode 100644 index 0000000..d331bfd --- /dev/null +++ b/render/server/db.mjs @@ -0,0 +1,59 @@ +// Driver setup shared by the render server and the jobs service, so both wait for neo4j the same way. +import neo4j from 'neo4j-driver'; +import { collectGraph } from './graph.mjs'; + +export const createDriver = (config, { poolSize = 16 } = {}) => + neo4j.driver(config.neo4j.uri, neo4j.auth.basic(config.neo4j.user, config.neo4j.password), { + maxConnectionPoolSize: poolSize, + connectionAcquisitionTimeout: 10_000, + disableLosslessIntegers: false, + }); + +// The compose stack may start the database slower than this container; 0 retries means wait forever. +export const waitForNeo4j = async (driver, config, log) => { + let attempt = 0; + for (;;) { + attempt += 1; + try { + const info = await driver.getServerInfo({ database: config.neo4j.database }); + log(`connected to ${config.neo4j.uri} (${info.agent}, protocol ${info.protocolVersion})`); + return; + } catch (err) { + const retriesLeft = config.neo4j.connectRetries === 0 ? 'unlimited' : config.neo4j.connectRetries - attempt; + log(`neo4j not reachable at ${config.neo4j.uri} (attempt ${attempt}, retries left: ${retriesLeft}): ${err.message}`); + if (config.neo4j.connectRetries !== 0 && attempt >= config.neo4j.connectRetries) { + throw err; + } + await new Promise((r) => setTimeout(r, config.neo4j.connectRetryDelayMs)); + } + } +}; + +/** Read transaction returning plain rows (driver integers and temporals converted like the API routes do). */ +export const readRows = async (driver, config, cypher, params = {}, { timeoutMs = config.neo4j.queryTimeoutMs } = {}) => { + const session = driver.session({ database: config.neo4j.database, defaultAccessMode: neo4j.session.READ }); + try { + const started = Date.now(); + const result = await session.executeRead((tx) => tx.run(cypher, params), { timeout: timeoutMs }); + const elapsedMs = Date.now() - started; + // Neo4j 5 can end a timed-out read as an empty stream instead of raising. + if (result.records.length === 0 && elapsedMs >= timeoutMs * 0.95) { + const err = new Error(`query timed out after ${elapsedMs} ms (limit ${timeoutMs} ms); the database may be busy`); + err.code = 'Neo.ClientError.Transaction.TransactionTimedOut'; + throw err; + } + return collectGraph(result.records).rows; + } finally { + await session.close(); + } +}; + +export const write = async (driver, config, cypher, params = {}, { timeoutMs = 600_000 } = {}) => { + const session = driver.session({ database: config.neo4j.database, defaultAccessMode: neo4j.session.WRITE }); + try { + const result = await session.executeWrite((tx) => tx.run(cypher, params), { timeout: timeoutMs }); + return result.summary.counters.updates(); + } finally { + await session.close(); + } +}; diff --git a/render/server/index.mjs b/render/server/index.mjs index c680217..70fc394 100644 --- a/render/server/index.mjs +++ b/render/server/index.mjs @@ -42,17 +42,15 @@ import { } from './registry.mjs'; import { createLodJobs } from './lod/jobs.mjs'; import { LodIndex } from './lod/index.mjs'; +import { createDriver, waitForNeo4j } from './db.mjs'; +import { STATS_IDS, overviewStats, readStats, readStatsAges } from './stats.mjs'; const here = path.dirname(fileURLToPath(import.meta.url)); const distDir = path.resolve(here, '..', 'dist'); const log = (...args) => console.log(new Date().toISOString(), ...args); -const driver = neo4j.driver(config.neo4j.uri, neo4j.auth.basic(config.neo4j.user, config.neo4j.password), { - maxConnectionPoolSize: 16, - connectionAcquisitionTimeout: 10_000, - disableLosslessIntegers: false, -}); +const driver = createDriver(config); const clampLimit = (value, fallback) => { const n = Number.parseInt(value ?? '', 10); @@ -79,25 +77,6 @@ const readGraph = async (cypher, params = {}, { timeoutMs = config.neo4j.queryTi } }; -const waitForNeo4j = async () => { - let attempt = 0; - for (;;) { - attempt += 1; - try { - const info = await driver.getServerInfo({ database: config.neo4j.database }); - log(`connected to ${config.neo4j.uri} (${info.agent}, protocol ${info.protocolVersion})`); - return; - } catch (err) { - const retriesLeft = config.neo4j.connectRetries === 0 ? 'unlimited' : config.neo4j.connectRetries - attempt; - log(`neo4j not reachable at ${config.neo4j.uri} (attempt ${attempt}, retries left: ${retriesLeft}): ${err.message}`); - if (config.neo4j.connectRetries !== 0 && attempt >= config.neo4j.connectRetries) { - throw err; - } - await new Promise((r) => setTimeout(r, config.neo4j.connectRetryDelayMs)); - } - } -}; - const app = express(); app.disable('x-powered-by'); app.use(express.json({ limit: '256kb' })); @@ -121,20 +100,55 @@ api.get( }), ); +// Whole-graph totals live on `Stats` nodes written by the jobs service (server/jobs/index.mjs); a route only +// runs the aggregate itself when no node exists yet (fresh database, jobs service not deployed). +const readRowsPlain = (cypher, params) => readGraph(cypher, params).then((g) => g.rows); +const precomputedStats = (id) => readStats(readRowsPlain, id).catch((err) => { + log(`stats ${id}: precomputed read failed, computing live: ${err.message}`); + return null; +}); + +// Forwards a control request to the jobs service; null when none is configured. +const jobsRequest = async (method, pathname, body) => { + if (!config.jobsUrl) return null; + const response = await fetch(`${config.jobsUrl}${pathname}`, { + method, + headers: body ? { 'content-type': 'application/json' } : undefined, + body: body ? JSON.stringify(body) : undefined, + signal: AbortSignal.timeout(15_000), + }); + const payload = await response.json().catch(() => ({})); + return { status: response.status, payload }; +}; + +const forwardToJobs = (method, pathname) => + wrap(async (req, res) => { + const r = await jobsRequest(method, pathname, method === 'POST' ? req.body ?? {} : undefined); + if (!r) return res.status(503).json({ error: 'no jobs service configured (RENDER_JOBS_URL); run server/jobs/index.mjs or call it directly', ids: STATS_IDS }); + res.status(r.status).json(r.payload); + }); + api.get( '/stats', wrap(async (_req, res) => { - const g = await readGraph( - `CALL { MATCH (c:Crate) RETURN count(c) AS crates } - CALL { MATCH (v:Version) RETURN count(v) AS versions } - CALL { MATCH ()-[r:DEPENDS_ON]->() RETURN count(r) AS dependsOn } - CALL { MATCH ()-[r:CALLS]->() RETURN count(r) AS invokes } - RETURN crates, versions, dependsOn, invokes`, - ); - res.json(g.rows[0] ?? {}); + const stored = await precomputedStats('overview'); + if (stored) return res.json(stored); + const started = Date.now(); + res.json({ ...(await overviewStats(readRowsPlain)), elapsedMs: Date.now() - started, computedAt: null, source: 'live' }); }), ); +api.get( + '/stats/status', + wrap(async (_req, res) => { + const stored = await readStatsAges(readRowsPlain); + const jobs = await jobsRequest('GET', '/api/stats/status').catch((err) => ({ status: 503, payload: { error: err.message } })); + res.json({ ids: STATS_IDS, stored, jobsUrl: config.jobsUrl || null, jobs: jobs?.payload ?? null }); + }), +); + +api.post('/stats/refresh', forwardToJobs('POST', '/api/stats/refresh')); + // A TEXT index on Crate.name (docker/queries.md) makes CONTAINS fast; without it this is a label scan. const searchCrates = async (q, limit) => readGraph( @@ -642,7 +656,7 @@ const nonNegInt = (v, fallback = 0) => { return Number.isFinite(n) && n >= 0 ? n : fallback; }; -/** Stats aggregate every version in scope (seconds for `all`), so cache them. */ +/** Live fallback when no `Stats` node exists: aggregates every version in scope (seconds for `all`), so cache it. */ const unsafeStatsCache = new Map(); const UNSAFE_STATS_TTL_MS = 60_000; @@ -650,10 +664,12 @@ api.get( '/unsafe/stats', wrap(async (req, res) => { const scope = unsafeScope(req.query.scope); + const stored = await precomputedStats(`unsafe:${scope}`); + if (stored) return res.json({ ...stored, cached: true }); const hit = unsafeStatsCache.get(scope); if (hit && Date.now() - hit.at < UNSAFE_STATS_TTL_MS) return res.json({ ...hit.value, cached: true }); const started = Date.now(); - const value = { ...(await unsafeStats(readRowsSlow, scope)), elapsedMs: Date.now() - started }; + const value = { ...(await unsafeStats(readRowsSlow, scope)), elapsedMs: Date.now() - started, computedAt: null, source: 'live' }; unsafeStatsCache.set(scope, { at: Date.now(), value }); res.json({ ...value, cached: false }); }), @@ -713,10 +729,12 @@ const cachedStats = (key, fn) => wrap(async (req, res) => { const scope = unsafeScope(req.query.scope); const cacheKey = `${key}:${scope}`; + const stored = await precomputedStats(cacheKey); + if (stored) return res.json({ ...stored, cached: true }); const hit = metaStatsCache.get(cacheKey); if (hit && Date.now() - hit.at < META_STATS_TTL_MS) return res.json({ ...hit.value, cached: true }); const started = Date.now(); - const value = { ...(await fn(readRowsSlow, scope)), elapsedMs: Date.now() - started }; + const value = { ...(await fn(readRowsSlow, scope)), elapsedMs: Date.now() - started, computedAt: null, source: 'live' }; metaStatsCache.set(cacheKey, { at: Date.now(), value }); res.json({ ...value, cached: false }); }); @@ -848,11 +866,16 @@ api.get( const REGISTRY_CACHE_TTL_MS = 10 * 60_000; const registryCache = new Map(); -const cachedRegistry = async (key, fn) => { +// `statsId` names the precomputed payload (stats.mjs) to prefer; the live path is the fallback. +const cachedRegistry = async (key, fn, statsId = null) => { + if (statsId) { + const stored = await precomputedStats(statsId); + if (stored) return { ...stored, cached: true }; + } const hit = registryCache.get(key); if (hit && Date.now() - hit.at < REGISTRY_CACHE_TTL_MS) return { ...hit.value, cached: true }; const started = Date.now(); - const value = { ...(await fn(readRowsSlow)), elapsedMs: Date.now() - started }; + const value = { ...(await fn(readRowsSlow)), elapsedMs: Date.now() - started, computedAt: null, source: 'live' }; registryCache.set(key, { at: Date.now(), value }); return { ...value, cached: false }; }; @@ -860,7 +883,7 @@ const cachedRegistry = async (key, fn) => { api.get( '/registry/stats', wrap(async (_req, res) => { - res.json(await cachedRegistry('stats', registryStats)); + res.json(await cachedRegistry('stats', registryStats, 'registry')); }), ); @@ -887,7 +910,7 @@ api.get( api.get( '/publishers/leaderboards', wrap(async (_req, res) => { - res.json(await cachedRegistry('leaderboards', (read) => publisherLeaderboards(read, 100))); + res.json(await cachedRegistry('leaderboards', (read) => publisherLeaderboards(read, 100), 'leaderboards')); }), ); @@ -1046,12 +1069,17 @@ api.get( // A build run elsewhere (CLI, another instance) changes built_at. const dbBuiltAt = meta?.built_at ? String(meta.built_at) : null; const idxBuiltAt = lodIndex?.meta?.built_at ? String(lodIndex.meta.built_at) : null; - if (!lodLoading && !lodJobs.status().running && ((lodIndex && dbBuiltAt !== idxBuiltAt) || (!lodIndex && dbBuiltAt))) { + // With a jobs service the worker runs there; its status is what the UI's Rebuild/Update buttons reflect. + const remote = await jobsRequest('GET', '/api/lod/status').catch((err) => ({ status: 503, payload: { error: err.message } })); + const jobStatus = remote && remote.status === 200 ? remote.payload : lodJobs.status(); + if (!lodLoading && !jobStatus.running && ((lodIndex && dbBuiltAt !== idxBuiltAt) || (!lodIndex && dbBuiltAt))) { log(`lod index: build changed (${idxBuiltAt} -> ${dbBuiltAt}), reloading`); loadLodIndex().catch(() => undefined); } res.json({ - ...lodJobs.status(), + ...jobStatus, + jobsUrl: config.jobsUrl || null, + jobsError: remote && remote.status !== 200 ? remote.payload?.error ?? `jobs service returned ${remote.status}` : null, meta, unpositionedCrates: unpositioned, index: lodIndex ? lodIndex.status() : null, @@ -1065,6 +1093,8 @@ api.get( api.post( '/lod/rebuild', wrap(async (req, res) => { + const remote = await jobsRequest('POST', '/api/lod/rebuild', req.body ?? {}); + if (remote) return res.status(remote.status).json(remote.payload); const iterations = Number.parseInt(req.body?.iterations ?? '', 10); res.status(202).json( lodJobs.rebuild({ @@ -1078,7 +1108,9 @@ api.post( api.post( '/lod/update', - wrap(async (_req, res) => { + wrap(async (req, res) => { + const remote = await jobsRequest('POST', '/api/lod/update', req.body ?? {}); + if (remote) return res.status(remote.status).json(remote.payload); res.status(202).json(lodJobs.update()); }), ); @@ -1157,11 +1189,12 @@ const server = app.listen(config.port, config.host, () => { }); // The compose stack may start the database slower than this container; /api/health reports the real state. -waitForNeo4j() +waitForNeo4j(driver, config, log) .then(() => { + if (config.jobsUrl) return log(`stats: served from Stats nodes, refreshes forwarded to ${config.jobsUrl}`); // Label scan of every version (tens of seconds under import load): warm the cache for the first Publishers visit. - cachedRegistry('stats', registryStats) - .then((s) => log(`registry stats warmed in ${s.elapsedMs} ms`)) + cachedRegistry('stats', registryStats, 'registry') + .then((s) => log(`registry stats ${s.source === 'precomputed' ? 'precomputed at ' + s.computedAt : 'warmed in ' + s.elapsedMs + ' ms'}`)) .catch((err) => log(`registry stats warm-up failed: ${err.message}`)); }) .catch((err) => { diff --git a/render/server/jobs/index.mjs b/render/server/jobs/index.mjs new file mode 100644 index 0000000..ac76641 --- /dev/null +++ b/render/server/jobs/index.mjs @@ -0,0 +1,196 @@ +// painter-jobs: the one process that writes derived data into the graph. It refreshes the dashboard totals +// (`Stats` nodes, see ../stats.mjs) on a timer and runs the Map (LOD) build/update worker on request, so render +// instances stay read-only and never pay for a whole-graph scan on a request. Same image as render, different +// command; the control API below is HTTP on $PORT and is meant for render (RENDER_JOBS_URL) and run scripts. +import express from 'express'; +import { config, describeConfig } from '../config.mjs'; +import { createDriver, readRows, waitForNeo4j, write } from '../db.mjs'; +import { createLodJobs } from '../lod/jobs.mjs'; +import { STATS_IDS, STATS_JOBS, STATS_SCHEMA_CYPHER, isStatsId, readStatsAges, writeStats } from '../stats.mjs'; + +const log = (...args) => console.log(new Date().toISOString(), ...args); + +const driver = createDriver(config, { poolSize: 4 }); +// Stats queries are label scans (up to ~95 s each on a cold database), so they get their own long timeout. +const read = (cypher, params) => readRows(driver, config, cypher, params, { timeoutMs: config.jobs.queryTimeoutMs }); +const writeCypher = (cypher, params) => write(driver, config, cypher, params); + +const startedAt = new Date().toISOString(); +let ready = false; + +// --- stats --- + +let statsRun = null; // { startedAt, reason, ids, done, current, results } +let statsLast = null; +let statsTimer = null; + +/** Runs the given stats jobs one after another (one scan at a time keeps the load on neo4j bounded). */ +const runStats = async ({ ids = STATS_IDS, reason = 'manual', skipFresherThanMs = 0 } = {}) => { + if (statsRun) { + const err = new Error(`a stats refresh is already running (${statsRun.done}/${statsRun.ids.length}, on ${statsRun.current})`); + err.status = 409; + throw err; + } + const run = { startedAt: new Date().toISOString(), reason, ids, done: 0, current: null, results: {} }; + statsRun = run; + log(`stats: refresh started (${reason}, ${ids.length} payloads)`); + try { + // Another instance (or a manual run) may have just written some: skip those within the freshness window. + const ages = skipFresherThanMs > 0 ? await readStatsAges(read).catch(() => ({})) : {}; + for (const id of ids) { + run.current = id; + const computedAt = ages[id]?.computedAt ? Date.parse(ages[id].computedAt) : 0; + if (skipFresherThanMs > 0 && Date.now() - computedAt < skipFresherThanMs) { + run.results[id] = { ok: true, skipped: true, computedAt: ages[id].computedAt }; + run.done += 1; + continue; + } + const started = Date.now(); + try { + const payload = await STATS_JOBS[id](read); + const elapsedMs = Date.now() - started; + await writeStats(writeCypher, id, payload, elapsedMs); + run.results[id] = { ok: true, elapsedMs }; + log(`stats: ${id} in ${elapsedMs} ms`); + } catch (err) { + run.results[id] = { ok: false, elapsedMs: Date.now() - started, error: err.message }; + log(`stats: ${id} failed after ${Date.now() - started} ms: ${err.message}`); + } + run.done += 1; + } + } finally { + run.current = null; + const failed = Object.values(run.results).filter((r) => !r.ok).length; + statsLast = { ...run, finishedAt: new Date().toISOString(), elapsedMs: Date.now() - Date.parse(run.startedAt), failed }; + statsRun = null; + log(`stats: refresh finished in ${statsLast.elapsedMs} ms, ${failed} failed`); + } + return statsLast; +}; + +const scheduleStats = () => { + const interval = config.jobs.statsIntervalMs; + if (interval <= 0) return log('stats: timer disabled (JOBS_STATS_INTERVAL_MS=0)'); + const tick = () => { + if (statsRun) return; + runStats({ reason: 'timer', skipFresherThanMs: interval / 2 }).catch((err) => log(`stats: timer run failed: ${err.message}`)); + }; + statsTimer = setInterval(tick, interval); + statsTimer.unref(); + tick(); + log(`stats: refreshing every ${Math.round(interval / 1000)} s`); +}; + +// --- lod --- + +const lodJobs = createLodJobs({ config, log }); + +const scheduleLodUpdate = () => { + const interval = config.jobs.lodUpdateIntervalMs; + if (interval <= 0) return; + const timer = setInterval(() => { + if (lodJobs.status().running) return; + try { + lodJobs.update({}); + } catch (err) { + log(`lod update: not started: ${err.message}`); + } + }, interval); + timer.unref(); + log(`lod: updating every ${Math.round(interval / 1000)} s`); +}; + +// --- control API --- + +const app = express(); +app.disable('x-powered-by'); +app.use(express.json({ limit: '64kb' })); + +const wrap = (fn) => (req, res, next) => fn(req, res, next).catch(next); + +const statsStatus = () => ({ running: statsRun ? { ...statsRun } : null, last: statsLast, intervalMs: config.jobs.statsIntervalMs, ids: STATS_IDS }); + +app.get( + '/api/health', + wrap(async (_req, res) => { + let error = null; + try { + await driver.getServerInfo({ database: config.neo4j.database }); + } catch (err) { + error = err.message; + } + res.status(error ? 503 : 200).json({ ok: !error, error, ready, startedAt, service: 'painter-jobs', config: describeConfig() }); + }), +); + +app.get( + '/api/status', + wrap(async (_req, res) => { + const stored = await readStatsAges(read).catch((err) => ({ error: err.message })); + res.json({ service: 'painter-jobs', startedAt, ready, stats: { ...statsStatus(), stored }, lod: lodJobs.status() }); + }), +); + +app.get('/api/stats/status', wrap(async (_req, res) => res.json({ ...statsStatus(), stored: await readStatsAges(read) }))); + +// Body: {ids?: string[]}; runs in the background, 202 with the run, 409 while one is in flight. +app.post( + '/api/stats/refresh', + wrap(async (req, res) => { + const requested = Array.isArray(req.body?.ids) ? req.body.ids.map(String) : null; + const unknown = (requested ?? []).filter((id) => !isStatsId(id)); + if (unknown.length) return res.status(400).json({ error: `unknown stats ids: ${unknown.join(', ')}`, ids: STATS_IDS }); + if (statsRun) return res.status(409).json({ error: 'a stats refresh is already running', ...statsStatus() }); + runStats({ ids: requested ?? STATS_IDS, reason: 'request' }).catch((err) => log(`stats: requested run failed: ${err.message}`)); + res.status(202).json(statsStatus()); + }), +); + +app.get('/api/lod/status', (_req, res) => res.json(lodJobs.status())); + +const lodStart = (kind) => + wrap(async (req, res) => { + const iterations = Number.parseInt(req.body?.iterations ?? '', 10); + const options = { iterations: Number.isFinite(iterations) && iterations > 0 ? iterations : undefined, reuseEdges: Boolean(req.body?.reuseEdges), resume: Boolean(req.body?.resume) }; + try { + res.status(202).json(kind === 'build' ? lodJobs.rebuild(options) : lodJobs.update(options)); + } catch (err) { + res.status(err.status ?? 500).json({ error: err.message, ...lodJobs.status() }); + } + }); + +app.post('/api/lod/rebuild', lodStart('build')); +app.post('/api/lod/update', lodStart('update')); + +app.use((err, _req, res, _next) => { + log(`error: ${err.message}`); + res.status(err.status ?? 500).json({ error: err.message }); +}); + +log('starting painter-jobs with config', JSON.stringify(describeConfig())); + +const server = app.listen(config.port, config.host, () => { + log(`control API on http://${config.host}:${config.port}`); +}); + +waitForNeo4j(driver, config, log) + .then(async () => { + for (const cypher of STATS_SCHEMA_CYPHER) await writeCypher(cypher); + ready = true; + scheduleStats(); + scheduleLodUpdate(); + }) + .catch((err) => { + log(`giving up on neo4j: ${err.message}`); + process.exit(1); + }); + +const shutdown = async (signal) => { + log(`received ${signal}, shutting down`); + if (statsTimer) clearInterval(statsTimer); + server.close(); + await driver.close(); + process.exit(0); +}; +process.on('SIGINT', () => shutdown('SIGINT')); +process.on('SIGTERM', () => shutdown('SIGTERM')); diff --git a/render/server/stats.mjs b/render/server/stats.mjs new file mode 100644 index 0000000..9528ec3 --- /dev/null +++ b/render/server/stats.mjs @@ -0,0 +1,68 @@ +// Dashboard totals are whole-graph aggregates (label scans of 3.2M versions, whole relationship types), so the +// jobs service computes them on a timer and stores each payload as JSON on a `(Stats {id})` node. The render +// server serves that node in milliseconds and only falls back to the live aggregate when no node exists yet. +// Payloads are JSON strings because Neo4j properties cannot hold lists of maps (topLinks, langs, boards). +import neo4j from 'neo4j-driver'; +import { unsafeStats } from './unsafe.mjs'; +import { buildStats, diffStats, nativeStats } from './metadata.mjs'; +import { publisherLeaderboards, registryStats } from './registry.mjs'; + +export const STATS_SCHEMA_CYPHER = ['CREATE CONSTRAINT const_stats_id IF NOT EXISTS FOR (s:Stats) REQUIRE s.id IS UNIQUE']; + +// The status bar's counts: four full scans, the slowest of the lot on the CALLS type. +export const overviewStats = async (read) => { + const rows = await read( + `CALL { MATCH (c:Crate) RETURN count(c) AS crates } + CALL { MATCH (v:Version) RETURN count(v) AS versions } + CALL { MATCH ()-[r:DEPENDS_ON]->() RETURN count(r) AS dependsOn } + CALL { MATCH ()-[r:CALLS]->() RETURN count(r) AS invokes } + RETURN crates, versions, dependsOn, invokes`, + ); + return rows[0] ?? {}; +}; + +/** Every precomputed payload: id -> (read) => payload. Ids are what the API routes look up. */ +export const STATS_JOBS = { + overview: (read) => overviewStats(read), + 'unsafe:latest': (read) => unsafeStats(read, 'latest'), + 'unsafe:all': (read) => unsafeStats(read, 'all'), + 'build:latest': (read) => buildStats(read, 'latest'), + 'build:all': (read) => buildStats(read, 'all'), + 'native:latest': (read) => nativeStats(read, 'latest'), + 'native:all': (read) => nativeStats(read, 'all'), + 'diffs:latest': (read) => diffStats(read, 'latest'), + 'diffs:all': (read) => diffStats(read, 'all'), + registry: (read) => registryStats(read), + leaderboards: (read) => publisherLeaderboards(read, 100), +}; + +export const STATS_IDS = Object.keys(STATS_JOBS); + +export const isStatsId = (id) => Object.hasOwn(STATS_JOBS, id); + +/** The stored payload for `id` with its provenance, or null when no job has written it yet. */ +export const readStats = async (read, id) => { + const rows = await read( + `MATCH (s:Stats {id: $id}) + RETURN s.payload AS payload, toString(s.computed_at) AS computedAt, s.elapsed_ms AS elapsedMs`, + { id }, + ); + const row = rows[0]; + if (!row?.payload) return null; + return { ...JSON.parse(row.payload), computedAt: row.computedAt, elapsedMs: row.elapsedMs, source: 'precomputed' }; +}; + +/** `computed_at` per id (ISO string), for freshness checks and the status endpoints. */ +export const readStatsAges = async (read) => { + const rows = await read(`MATCH (s:Stats) RETURN s.id AS id, toString(s.computed_at) AS computedAt, s.elapsed_ms AS elapsedMs, s.bytes AS bytes`); + return Object.fromEntries(rows.map((r) => [r.id, { computedAt: r.computedAt, elapsedMs: r.elapsedMs, bytes: r.bytes }])); +}; + +export const writeStats = (write, id, payload, elapsedMs) => { + const json = JSON.stringify(payload); + return write( + `MERGE (s:Stats {id: $id}) + SET s.payload = $payload, s.computed_at = datetime(), s.elapsed_ms = $elapsedMs, s.bytes = $bytes`, + { id, payload: json, elapsedMs: neo4j.int(elapsedMs), bytes: neo4j.int(Buffer.byteLength(json)) }, + ); +}; diff --git a/render/src/api.ts b/render/src/api.ts index 5a99193..eaa4f4c 100644 --- a/render/src/api.ts +++ b/render/src/api.ts @@ -183,7 +183,15 @@ export interface UnsafeRow { share: number; } -export interface UnsafeStats { +/** Where a dashboard total came from: a `Stats` node the jobs service wrote (`computedAt`), or a live aggregate. */ +export interface StatsMeta { + computedAt?: string | null; + source?: 'precomputed' | 'live'; + elapsedMs?: number; + cached?: boolean; +} + +export interface UnsafeStats extends StatsMeta { scope: UnsafeScope; versions: number; scanned: number; @@ -283,7 +291,7 @@ export interface LinksRow { sample: string[]; } -export interface BuildStats { +export interface BuildStats extends StatsMeta { scope: UnsafeScope; versions: number; scanned: number; @@ -341,7 +349,7 @@ export interface NativeRow { export type NativeSort = 'lines' | 'sourceLines' | 'files' | 'bytes' | 'largest' | 'vendorDirLines' | 'prebuilt' | 'name'; -export interface NativeStats { +export interface NativeStats extends StatsMeta { scope: UnsafeScope; versions: number; scanned: number; @@ -412,7 +420,7 @@ export interface DiffVersion extends DiffRow { files: DiffFile[] | null; } -export interface DiffStats { +export interface DiffStats extends StatsMeta { scope: UnsafeScope; versions: number; scanned: number; @@ -443,7 +451,7 @@ export interface DiffTopQuery { offset: number; } -export interface RegistryStats { +export interface RegistryStats extends StatsMeta { users: number; teams: number; owns: number; @@ -553,7 +561,7 @@ export interface TopPublisherRow { sample: string[]; } -export interface PublisherLeaderboards { +export interface PublisherLeaderboards extends StatsMeta { byCrates: ProlificRow[]; byUploads: ProlificRow[]; owners: TopOwnerRow[]; @@ -709,7 +717,7 @@ export interface TopResult { meta: { elapsedMs: number; sort: Sort; scope: UnsafeScope }; } -export interface Stats { +export interface Stats extends StatsMeta { crates: number; versions: number; dependsOn: number; diff --git a/render/src/components/BuildView.tsx b/render/src/components/BuildView.tsx index c129bc3..67dffbb 100644 --- a/render/src/components/BuildView.tsx +++ b/render/src/components/BuildView.tsx @@ -149,6 +149,7 @@ export function BuildView({ busy, onBusy, onError, onOpenCrate }: Props) {
{fmt(stats?.cratesInRegistry)} of {fmt(stats?.crates)} crates · {fmt(stats?.versionsMissing)} versions not in the dump · {fmt(stats?.versionUploads)} uploads with a recorded uploader
+ {freshness(stats) &&
{freshness(stats)}
} diff --git a/render/src/components/UnsafeView.tsx b/render/src/components/UnsafeView.tsx index 4c3c63c..9b3f13a 100644 --- a/render/src/components/UnsafeView.tsx +++ b/render/src/components/UnsafeView.tsx @@ -1,4 +1,5 @@ import { Fragment, useCallback, useEffect, useMemo, useState } from 'react'; +import { freshness } from './meta'; import { api, ApiError, @@ -298,6 +299,7 @@ export function UnsafeView({ busy, onBusy, onError, onOpenCrate }: Props) {
{fmt(stats?.unreadable)} unreadable · {fmt(stats?.files)} files · {fmt(stats?.lines)} lines
+ {freshness(stats) &&
{freshness(stats)}
}
with any unsafe
diff --git a/render/src/components/meta.tsx b/render/src/components/meta.tsx index fdbe90d..0dc17cd 100644 --- a/render/src/components/meta.tsx +++ b/render/src/components/meta.tsx @@ -1,5 +1,5 @@ import { useCallback, useEffect, useMemo, useState, type ReactNode } from 'react'; -import { ApiError, type UnsafeScope } from '../api'; +import { ApiError, type StatsMeta, type UnsafeScope } from '../api'; export const fmt = (n: number | null | undefined) => (n === null || n === undefined ? '—' : n.toLocaleString()); export const pct = (part: number | null | undefined, whole: number | null | undefined) => @@ -18,6 +18,25 @@ export const fmtDateTime = (iso: string | null | undefined) => { const d = new Date(iso); return Number.isNaN(d.getTime()) ? iso : d.toLocaleString(); }; +export const fmtAgo = (iso: string) => { + const ms = Date.now() - Date.parse(iso); + if (!Number.isFinite(ms) || ms < 0) return fmtDateTime(iso); + const m = Math.round(ms / 60_000); + if (m < 1) return 'just now'; + if (m < 90) return `${m} min ago`; + const h = Math.round(m / 60); + if (h < 36) return `${h} h ago`; + return `${Math.round(h / 24)} d ago`; +}; + +/** "totals computed 5 min ago" for a precomputed payload, or that they were aggregated on request. */ +export const freshness = (s: StatsMeta | null | undefined) => { + if (!s) return null; + if (s.computedAt) return `totals computed ${fmtAgo(s.computedAt)}`; + if (s.source === 'live') return `totals computed live (no jobs service)${s.elapsedMs ? ` in ${s.elapsedMs.toLocaleString()} ms` : ''}`; + return null; +}; + export const fmtCompact = (n: number | null | undefined) => { if (n === null || n === undefined) return '—'; if (n >= 1e9) return `${(n / 1e9).toFixed(2)}B`; @@ -87,9 +106,10 @@ export function RowsSelect({ value, onChange }: { value: number; onChange: (n: n ); } -export function CoverageTile({ scope, scanned, versions, note, title }: { scope: UnsafeScope; scanned?: number; versions?: number; note?: ReactNode; title: string }) { +export function CoverageTile({ scope, scanned, versions, note, title, stats }: { scope: UnsafeScope; scanned?: number; versions?: number; note?: ReactNode; title: string; stats?: StatsMeta | null }) { + const fresh = freshness(stats); return ( -
+
scan coverage · {scope === 'latest' ? 'latest versions' : 'all versions'}
{pct(scanned, versions)} {fmt(scanned)} of {fmt(versions)} @@ -98,6 +118,7 @@ export function CoverageTile({ scope, scanned, versions, note, title }: { scope:
{note &&
{note}
} + {fresh &&
{fresh}
}
); } diff --git a/src/migrate.rs b/src/migrate.rs index d9dd00f..562d18a 100644 --- a/src/migrate.rs +++ b/src/migrate.rs @@ -23,6 +23,9 @@ pub enum Error { MissingColumn(&'static str), } +/// Derived nodes the rebuild drops on purpose: the render jobs service recomputes them on its timer. +const DROPPED_LABELS: &[&str] = &["Stats"]; + const NODE_PROPERTIES: &[(&str, &[&str])] = &[ ( "Crate", @@ -171,6 +174,11 @@ async fn check_schema(graph: &Graph) -> Result<(), Error> { while let Some(row) = rows.next().await? { let labels = need_strs(&row, "nodeLabels")?; let prop = row.get::("propertyName"); + if let [label] = labels.as_slice() + && DROPPED_LABELS.contains(&label.as_str()) + { + continue; + } let known = match labels.as_slice() { [label] => NODE_PROPERTIES .iter()