diff --git a/render/README.md b/render/README.md index 889984e..3b29888 100644 --- a/render/README.md +++ b/render/README.md @@ -76,8 +76,9 @@ Locally: `npm run start:jobs` (port 8081) next to `RENDER_JOBS_URL=http://127.0. Create the constraints and indexes in `docker/queries.md`; views are slow or unusable without them: -- `idx_crate_name_text`: partial-name search. -- `const_symbol_name`: Callers view (one seek per function name on `Symbol.name`, then its callers). +- `idx_crate_name_text`: partial-name hints in the search dropdowns. +- `const_symbol_name`, `idx_symbol_name_text`: Callers view (one seek per function name on `Symbol.name`, + then its callers; the TEXT index backs the function search). - `idx_has_build_script`, `idx_version_links`, `idx_has_native_code`, `idx_vendored_native`, `idx_diff_lines_added`: Build scripts / Native code / Diffs leaderboards. - Registry indexes (`idx_user_login`, `idx_published_at`, `idx_owns_since`, @@ -91,24 +92,82 @@ Default and Deps work as soon as the graph has data; Map needs the precompute be Graph views (Default, Deps, Map): wheel/pinch zooms, drag pans; click a node or edge for properties. -**Default**: crates, versions, and `VERSION_OF` / `DEPENDS_ON` / `INVOKES` edges around a crate name -(case-insensitive substring, prefix first; Load takes the top match). "Latest" is -`Version.latest` when present, else the highest semver triple. `INVOKES` is not stored: the server -folds calls into one synthetic edge per (`Version`, `Crate`) with a `count` of distinct calls and up -to ten sample `targets` (ids start with `agg:`). The Cypher panel returns raw relationships and -merges nodes, relationships, and paths into the canvas. - -**Deps**: crates only; an edge A -> B means some version of A depends on B, and its width is the -number of versions that declare it. An exact crate name is the only root; otherwise, with 3+ -characters, every crate containing the text is a root. The walk stops at max depth (0 = unlimited) -or max edges (default 5000); the status bar says which. Full-depth chains of big crates hit the cap -fast because every historical version is folded in, so use depth 2-3. - -**Map**: the whole ecosystem on a precomputed layout, drawn at a level of detail that keeps the -element count under max visible (default 600); colour is Louvain community. - -**Callers**: a table of every crate version and function that calls a given function, directly or -transitively, with the call chain per row. Requirement checks are applied per hop. +**Searching** (`server/match.mjs`, `src/match.ts`): every search box and table filter takes an exact +name, case-insensitive. Only `*` (any run of characters) and `?` (one character) make it a pattern: +`tokio*`, `*-sys`, `serde_?son`. The hint dropdowns still offer partial matches as you type, and +picking one sets that exact name. A pattern in a graph view's box (or the Callers function box) opens +the best match, by downloads (Map: importance, Callers: callers). In the table views (Unsafe, Build +scripts, Native code, Diffs, Releases, a publisher's crates, Callers) it returns every matching row, +and in the Publishers login box it lists every matching login. The version tables resolve the crate +names first (`crateNameScope`: one index lookup, or a regex over the crate names when at most 5,000 +match) and start from those crates, so an exact or rare filter does not walk every version. + +**Default**: one crate's overview (`GET /api/graph/overview/:name`, `server/overview.mjs`; the name +is exact, or a `*` / `?` pattern). One bounded Cypher round trip returns the +crate with degree-store counts, what its latest version depends on (folded to one `DEPENDS_ON` per +crate: strongest `kind`, `optional` only if every declaration is, all requirements, `via` = the +declaring version; ids `dep:`), its 12 newest versions, the crates its compiled code calls into +(`calls` on the dependency edge when declared, else a red `INVOKES` edge to a `callee` node; ids +`agg:`; self-calls dropped), and with the dependents option its 25 most-downloaded dependents from +`LOD_DEP`. The client (`src/overview.ts`) lays this out itself on NVL's `free` layout: root in the +middle, dependencies fanned right (normal / optional / called on the inner arc, build / dev outside, +size = downloads, colour = kind), dependents left, versions as an oldest → latest strip along the +bottom joined by a canvas-only `PRECEDES` chain, crates.io accounts on top. Captions are DOM labels +(`components/OverviewLabels.tsx`) that follow the canvas, with a detail line (requirement, kind, +calls; versions using it; publish date and ⊘ ☢ ⚙ C Δ badges) once zoomed past 0.7. Dragging keeps +positions; `layout` can switch back to force directed. The details panel pages bounded expansions +(`Latest deps`, `Dependents +N` 25 at a time, `Versions +N` 12 at a time, `Calls`, `Owners`, +`Publishers`), double-click adds a crate's or a version's dependencies, and new nodes fan out from +the node that was expanded. Inbound callers are not drawn (4-15 s even bounded); the panel shows +how many functions are called from other crates and the Callers view lists who. The Cypher panel +returns raw relationships and merges nodes, relationships, and paths into the canvas. + +**Deps**: one crate's dependency tree, root on top and one row per hop (layered layout, not force). +By default it follows the root's current version (`Crate.default_version`, else `Version.latest`) +and its required normal dependencies, two hops deep, at most 300 crates. Toolbar toggles add +optional (drawn fainter), build (orange), and dev (purple) edges; "any version" switches to the +Map's crate-level `LOD_DEP` edges (width = versions declaring it); "dependents" walks `LOD_DEP` +backwards, keeping the 30 best-downloaded dependents of the root and 3 of each of those, and in +current-version mode checks each candidate's current version. Node size is crate-level in-degree. +Edges to a crate already reached by a shorter path are drawn faint unless they touch the selection. +The root's details say which declarations the toggles leave out (tokio: 20 dev, 15 optional). When the budget or depth cuts a +crate's children, it shows "+N"; double-click (or Show +N) fetches them, double-click again folds +them. Clicking a crate highlights its path from the root. The sidebar has the path, children, and +the legend. Reverse mode and node sizes need `LOD_DEP` (the Map build); without it, they show 0. + +**Map**: the whole ecosystem on a precomputed layout. Each viewport shows the most important crates +in it as labelled landmarks (importance = log downloads + 1.5 log dependents, picked in global order +with a minimum on-screen spacing, so a landmark stays one at every deeper zoom), a smoothed density +shading for everything else, and the largest Louvain communities as translucent blobs labelled by +their dominant crates ("serde · serde_json · thiserror"). Only the 16 largest communities (by +downloads) get a colour; the rest are grey. The sidebar lists them as a clickable legend. Clicking a +crate shows downloads, dependents, dependencies, its most important neighbours (jump or "deps" to +open the Deps view), and draws its edges: orange from crates that use it, blue to what it depends +on. Other edges are drawn only when there are few enough to read (<= 220). The toolbar "density" +preset (sparse / normal / dense) sets how many landmarks fit on a screen. The view opens on the +populated core (1.5-98.5 percentile box); "Whole map" includes the ring of isolated crates. +`/api/lod/viewport` answers from memory in ~10-50 ms; `/api/lod/neighbours` and +`/api/lod/communities` are computed from the same in-memory index, no database query. + +**Callers**: who calls a function. One search box takes `crate::path::fn`, `crate::` (that crate's +most-called functions), or a bare name, with suggestions grouped by crate and ranked by callers. +Suggestions are partial; with `*` / `?` the query is a full-name pattern (`serde_json::*from_str`: +a regex over that crate's symbols; a global pattern seeds from the TEXT index on its longest literal, +which must be at least 3 characters, then filters with the regex). Enter without a picked suggestion +opens plain text only as an exact function name ("no function named X" otherwise) and a pattern's +match with the most callers. The result is aggregated per caller crate (ordered by downloads, with +the calling versions and the number of calling functions) and paged server-side (100 crates a page, +up to 500; sort by downloads / functions / versions / crate). The server caches the per-version +aggregate (no function names) per (function, pin, cap) for 10 minutes, 20 entries, so paging, +sorting, and the caller crate filter (exact name, or a pattern like `tokio*`, with hints from the +cached list) never re-run the fan-in; calling function names (40 per crate) are fetched for the +page's crates only. A crate row opens into its calling functions, and a function opens into its own +callers one level further out (paged 50 at a time). The callee version is pinned to the latest (newest stable if that is a pre-release) by default: a caller counts +when its `DEPENDS_ON` requirement admits that version (`unverified` when it has no direct dependency, +dropped when the requirement excludes it). The graph pane draws the current page's crates (edge width = calling +functions, node size = downloads); `calls` flips to what the function calls. Hot symbols are answered +from the first 15,000 calling functions with an "expand all" link, since every calling function is a +random read. `focus` re-roots on any function; `graph` / `deps` open the crate in those views. **Unsafe**, **Build scripts**, **Native code**, **Diffs**: leaderboards over the `generate-metadata` properties (`docker/queries.md` lists them). Scope is latest versions (index-backed) or all @@ -140,6 +199,11 @@ Stored in the database so every render instance shares it: declare it - `(:LodMeta {id:'default'})`: counts, bounds, timings, `built_at` +The render server loads all of that (plus `Crate.downloads`) into memory at start: ~4 min against a +remote database, during which the Map reports "loading". It then writes a JSON snapshot to +`RENDER_LOD_INDEX_CACHE` (default `/tmp/painter-lod-index.json`, ~65 MB, empty disables it) keyed on +`built_at`, so a restart loads in a couple of seconds until the next build or update. + A full build aggregates every `DEPENDS_ON`, runs Louvain and a single-threaded ForceAtlas2, and writes it all back. On 340k crates / 37M `DEPENDS_ON` it takes ~30 min and ~2.5 GB, almost all of it the layout. **Update** only places crates with no position yet (imported after the last build) at the @@ -177,11 +241,17 @@ obvious ones: | Route | What | | --- | --- | | `GET /api/health` | connection state and effective config, no secrets | -| `GET /api/graph/crate/:name` | Default load; `name` may be partial, `meta.resolvedName` says what was picked | +| `GET /api/graph/overview/:name` | Default load; `parts=deps,versions,calls,dependents`, `version` + `from=version`, `skip`, `limit`, `root=1`; nodes carry `role` / `counts`, `meta.overview` the totals | +| `GET /api/graph/crate/:name` | older raw load (Crate, Versions, their edges); `name` is exact or a `*` / `?` pattern, `meta.resolvedName` says what was picked | | `POST /api/graph/expand` | neighbours of a node (`VERSION_OF`, `DEPENDS_ON`, `INVOKES`, `OWNS`, `PUBLISHED`) | -| `GET /api/graph/deps/chain` | Deps view; `meta.roots/depth/truncated` | +| `GET /api/deps/tree?crate&direction=out\|in&mode=latest\|any&kinds&optional&depth&limit` | Deps tree keyed by crate name: nodes with `depth/downloads/dependents/hidden/unexpanded`, edges with `kinds/optional/req/versions` | +| `GET /api/deps/expand?crate&...` | one crate's direct neighbours for the Deps view (same options) | | `GET /api/lod/*` | Map status, viewport, locate, search; viewport is 503 until a build is loaded | -| `GET /api/callers?crate&fn&match&version&depth&limit` | Callers rows; `match=exact\|contains` | +| `GET /api/symbols/search?q&limit` | function search (`crate::` prefix scopes it); `symbols` grouped by `crate`, `crates` prefix matches; `*` / `?` make it a pattern (`pattern`, `hint`) | +| `GET /api/symbols/resolve?q` | what Enter opens: the exact symbol for plain text, the most-called pattern match, or `symbol: null` | +| `GET /api/symbols/callers?fn&versions&cap&localCrate&localVersions&offset&limit&sort&desc&filter` | one page of caller crates (cached aggregate), per-version requirement status, names for the page's crates; totals for the filtered set, `meta.hasMore`, `meta.truncated` when `total > cap` | +| `GET /api/symbols/callers/crates?fn&versions&cap&q&limit` | caller crate names containing `q` (filter hints), off the same cached aggregate | +| `GET /api/symbols/callees?fn&crate&version` | what a function calls, in the chosen or newest compiled copy | | `GET /api/{unsafe,build,native,diffs}/{stats,top,crate/:name}` | leaderboard tiles, sorted/filtered rows (`hasMore` for paging), one crate's per-version history; stats come from the jobs service's `Stats` nodes (`computedAt`), live + cached 60 s without one | | `GET /api/stats/status`, `POST /api/stats/refresh` | stored `Stats` ages; refresh forwarded to `RENDER_JOBS_URL` (503 without one) | | `GET /api/registry/*`, `/api/publishers/*`, `/api/releases*`, `/api/supply/*` | registry, publisher, release, and supply-chain data | diff --git a/render/server/callers.mjs b/render/server/callers.mjs index f0aa47f..61e6b97 100644 --- a/render/server/callers.mjs +++ b/render/server/callers.mjs @@ -4,6 +4,8 @@ // A callsite whose symbol belongs to a crate other than the caller's is a monomorphised copy in the caller's binary, // so only that same version can call it. +import { hasWildcard, longestLiteral, nameTester, wildcardRegex } from './match.mjs'; + // Mirrors analysis.rs BLOCKED_CRATES. const BLOCKED_CRATES = new Set(['core', 'std', 'alloc']); @@ -93,99 +95,473 @@ export const requirementMatches = (requirement, version) => { return true; }; -// Starts from the Symbol.name uniqueness index: one seek per target plus its callers. -const GLOBAL_HOP = ` - UNWIND $targets AS tg - MATCH (s:Symbol {name: tg})-[:IN]->(t:Crate) - MATCH (s)<-[:CALLS]-(f:Fn)<-[:HAS_FN]-(v:Version) - MATCH (f)-[:IS]->(cs:Symbol) + +// --------------------------------------------------------------------------- +// Symbol search: one box for `crate::path::fn`, `crate::`, or a bare name. +// +// Suggestions are partial (CONTAINS) for plain text. A query with `*` / `?` is a full-name pattern +// (server/match.mjs): `crate::pattern` is a regex over that crate's symbols, a global pattern seeds +// from an index-backed CONTAINS on its longest literal and filters that with the regex. + +// Symbol paths spell `my-crate` as `my_crate`; accept either. +export const crateSpellings = (name) => [...new Set([name, normCrate(name), name.replace(/_/g, '-')])]; + +const SCOPED_SEARCH = ` + MATCH (t:Crate) WHERE t.name IN $names + MATCH (t)<-[:IN]-(s:Symbol) + WHERE $q = '' OR s.name CONTAINS $q + WITH t, s, COUNT { (s)<-[:CALLS]-() } AS calls + ORDER BY s.name ENDS WITH $tail DESC, calls DESC, size(s.name) + RETURN t.name AS crate, s.name AS name, calls LIMIT $limit`; + +// TEXT index idx_symbol_name_text makes CONTAINS an index scan; the scan cap keeps a common word +// (`new`, `fmt`) from walking all 4.6M symbols before ranking. +const GLOBAL_SEARCH = ` + MATCH (s:Symbol) WHERE s.name CONTAINS $q + WITH s LIMIT $scan + WITH s, COUNT { (s)<-[:CALLS]-() } AS calls + ORDER BY s.name ENDS WITH $tail DESC, calls DESC, size(s.name) + LIMIT $limit + MATCH (s)-[:IN]->(t:Crate) + RETURN t.name AS crate, s.name AS name, calls`; + +const CRATE_PREFIX = ` + MATCH (c:Crate) WHERE c.name STARTS WITH $q OR c.name STARTS WITH $alt + WITH c ORDER BY size(c.name), c.name LIMIT $limit + RETURN c.name AS name`; + +// A crate holds at most a few thousand called symbols (syn ~3.1k), so the regex walks them all; +// `$scan` only bounds the matches ranked. +const SCOPED_PATTERN = ` + MATCH (t:Crate) WHERE t.name IN $names + MATCH (t)<-[:IN]-(s:Symbol) + WHERE s.name =~ $re + WITH t, s LIMIT $scan + WITH t, s, COUNT { (s)<-[:CALLS]-() } AS calls + ORDER BY calls DESC, size(s.name), s.name + RETURN t.name AS crate, s.name AS name, calls LIMIT $limit`; + +// CONTAINS is case-sensitive, the regex is not: the seed decides what the pattern can find. +const GLOBAL_PATTERN = ` + MATCH (s:Symbol) WHERE s.name CONTAINS $lit + WITH s LIMIT $scan + WITH s WHERE s.name =~ $re + WITH s, COUNT { (s)<-[:CALLS]-() } AS calls + ORDER BY calls DESC, size(s.name), s.name + LIMIT $limit + MATCH (s)-[:IN]->(t:Crate) + RETURN t.name AS crate, s.name AS name, calls`; + +const EXACT_SYMBOL = ` + MATCH (s:Symbol {name: $name}) + OPTIONAL MATCH (s)-[:IN]->(t:Crate) + RETURN s.name AS name, t.name AS crate, COUNT { (s)<-[:CALLS]-() } AS calls + LIMIT 1`; + +/** Shortest literal a global pattern needs, so it seeds from the TEXT index instead of scanning. */ +export const MIN_PATTERN_LITERAL = 3; +const SEARCH_SCAN = 1500; +const SCOPED_PATTERN_SCAN = 5000; + +const patternHint = `a pattern needs a run of at least ${MIN_PATTERN_LITERAL} plain characters to search from (e.g. serde_json::*from_str or *::from_str)`; + +/** Symbols matching a wildcard query, most callers first. */ +const searchPattern = async (read, int, text, { limit, scan }) => { + const sep = text.indexOf('::'); + const head = sep > 0 ? text.slice(0, sep) : ''; + if (head && !hasWildcard(head)) { + const re = wildcardRegex(normCrate(head) + text.slice(sep)); + const symbols = await read(SCOPED_PATTERN, { names: crateSpellings(head), re, scan: int(SCOPED_PATTERN_SCAN), limit: int(limit) }); + return { symbols, crates: [], scope: head, pattern: true }; + } + const lit = longestLiteral(text); + if (lit.length < MIN_PATTERN_LITERAL) return { symbols: [], crates: [], scope: null, pattern: true, hint: patternHint }; + const symbols = await read(GLOBAL_PATTERN, { lit, re: wildcardRegex(text), scan: int(scan), limit: int(limit) }); + return { symbols, crates: [], scope: null, pattern: true, seed: lit, scan }; +}; + +export const searchSymbols = async (read, int, q, { limit = 30, scan = SEARCH_SCAN } = {}) => { + const started = Date.now(); + const text = q.trim(); + if (!text) return { symbols: [], crates: [], scope: null, elapsedMs: 0 }; + if (hasWildcard(text)) return { ...(await searchPattern(read, int, text, { limit, scan })), elapsedMs: Date.now() - started }; + const sep = text.indexOf('::'); + const tail = text.split('::').pop(); + if (sep > 0) { + const head = text.slice(0, sep); + const rest = text.slice(sep + 2); + const scoped = await read(SCOPED_SEARCH, { names: crateSpellings(head), q: rest, tail, limit: int(limit) }); + if (scoped.length || rest === '') return { symbols: scoped, crates: [], scope: head, elapsedMs: Date.now() - started }; + } + const [symbols, crates] = await Promise.all([ + read(GLOBAL_SEARCH, { q: text, tail, scan: int(scan), limit: int(limit) }), + // A bare word is often a crate name: offer `crate::` so the next keystroke scopes the search. + sep > 0 + ? Promise.resolve([]) + : read(CRATE_PREFIX, { q: text, alt: text.replace(/_/g, '-'), limit: int(5) }).then((rows) => rows.map((r) => r.name)), + ]); + return { symbols, crates, scope: null, elapsedMs: Date.now() - started }; +}; + +/** + * What Enter opens without a picked suggestion: plain text only an exact (case-sensitive, it is an + * index seek) full symbol name; a wildcard query the match with the most callers. + */ +export const resolveSymbol = async (read, int, q) => { + const started = Date.now(); + const text = q.trim(); + if (!text) return { symbol: null, pattern: false, elapsedMs: 0 }; + if (!hasWildcard(text)) { + const [row] = await read(EXACT_SYMBOL, { name: text }); + const symbol = row ? { name: row.name, crate: row.crate ?? crateOf(row.name), calls: row.calls } : null; + return { symbol, pattern: false, elapsedMs: Date.now() - started }; + } + const r = await searchPattern(read, int, text, { limit: 30, scan: SEARCH_SCAN }); + return { symbol: r.symbols[0] ?? null, pattern: true, matches: r.symbols.length, hint: r.hint ?? null, elapsedMs: Date.now() - started }; +}; + +// --------------------------------------------------------------------------- +// Callers, aggregated per crate and served in pages. +// +// One aggregate pass per (symbol, pinned versions, local scope, cap) returns per caller version only +// its crate, downloads, latest flag, calling-function count, and DEPENDS_ON requirements; the semver +// check runs here and the result is cached (CACHE_SIZE entries, CACHE_TTL_MS). Pages sort and filter +// that in memory, then fetch calling function names for the page's crates only. + +/** + * Default cap on the calling `Fn` nodes expanded for one symbol. Each one is a random read of the + * Fn and its Version: ~1 s warm but 10-25 s cold per 50k on the production disk, so hot symbols + * (`serde_json::de::from_str` 48k, `Deserializer::peek_error` 154k) answer from a prefix first and + * the UI offers to expand the rest (`cap` up to MAX_FN_CAP). + */ +export const DEFAULT_FN_CAP = 15_000; +export const MAX_FN_CAP = 250_000; +/** Function names returned per caller crate; the rest is `functionCount`. */ +const FUNCTIONS_PER_CRATE = 40; +export const DEFAULT_PAGE = 100; +export const MAX_PAGE = 500; +const CACHE_SIZE = 20; +const CACHE_TTL_MS = 10 * 60_000; +export const CALLER_SORTS = ['downloads', 'functions', 'versions', 'crate']; + +// Symbol side: one seek on const_symbol_name, then the CALLS fan-in (the expensive part for hot +// symbols: every Fn and its Version are random reads). `getDegree` gives the true total for free. +// One row per caller version, so at most $cap rows. +const CALLERS_AGG = ` + MATCH (s:Symbol {name: $name})-[:IN]->(t:Crate) + WITH s, t, COUNT { (s)<-[:CALLS]-() } AS total + MATCH (s)<-[:CALLS]-(f:Fn) + WITH t, total, f LIMIT $cap + MATCH (f)<-[:HAS_FN]-(v:Version) + WITH t, total, v, count(f) AS fns + MATCH (cc:Crate)-[:VERSION_OF]->(v) OPTIONAL MATCH (v)-[d:DEPENDS_ON]->(t) - WITH v, cs, s, t, collect(DISTINCT d.requirement) AS reqs - RETURN v.name AS name, v.version AS version, cs.name AS callsite, s.name AS target, t.name AS crate, reqs`; - -const LOCAL_HOP = ` - UNWIND $locals AS l - UNWIND l.targets AS tg - MATCH (s:Symbol {name: tg})-[:IN]->(t:Crate) - MATCH (s)<-[:CALLS]-(f:Fn)<-[:HAS_FN]-(v:Version {name: l.name, version: l.version}) + WITH t, total, cc, v, fns, collect(DISTINCT d.requirement) AS reqs + RETURN t.name AS targetCrate, total, cc.name AS crate, cc.downloads AS downloads, v.version AS version, + coalesce(v.latest, false) AS latest, fns, reqs + LIMIT $cap`; + +// Version side, for a symbol that is a monomorphised copy inside the caller's own binary: only the +// versions that contain it can call it, and those are few, so start from them. +const CALLERS_LOCAL_AGG = ` + MATCH (s:Symbol {name: $name}) + WITH s, COUNT { (s)<-[:CALLS]-() } AS total + UNWIND $versions AS ver + MATCH (cc:Crate {name: $crate})-[:VERSION_OF]->(v:Version {version: ver}) + MATCH (v)-[:HAS_FN]->(f:Fn)-[:CALLS]->(s) + WITH total, cc, v, count(f) AS fns + RETURN $crate AS targetCrate, total, cc.name AS crate, cc.downloads AS downloads, v.version AS version, + coalesce(v.latest, false) AS latest, fns, [] AS reqs + LIMIT $cap`; + +// Calling function names for one page: the same capped fan-in as CALLERS_AGG (so the same callers), +// narrowed to the page's kept versions before the Fn -> Symbol hop. `versions` = in how many of the +// crate's calling versions the function calls it. +const CALLERS_NAMES = ` + MATCH (s:Symbol {name: $name})<-[:CALLS]-(f:Fn) + WITH f LIMIT $cap + MATCH (f)<-[:HAS_FN]-(v:Version) + WHERE v.name IN $crates AND v.name + '@' + v.version IN $keys MATCH (f)-[:IS]->(cs:Symbol) - RETURN v.name AS name, v.version AS version, cs.name AS callsite, s.name AS target, t.name AS crate, [] AS reqs`; - -const versionKey = (name, version) => `${name}@${version}`; - -export const findCallers = async (read, { crate, version = null, targets, maxDepth, limit }) => { - const rows = []; - const seen = new Set(); - // Frontier entries: { fn, crate, version, local, path }. `local` = only the - // version that contains it can call it (foreign monomorphised copy). - let frontier = targets.map((fn) => ({ fn, crate, version, local: false, path: [fn] })); - let depth = 0; - let truncated = false; - let unverified = 0; - let rejected = 0; + WITH v.name AS crate, cs.name AS fn, count(*) AS nv + ORDER BY crate, nv DESC, fn + WITH crate, count(*) AS functionCount, collect({name: fn, versions: nv})[0..$per] AS functions + RETURN crate, functionCount, functions + LIMIT $limit`; - while (frontier.length && depth < maxDepth && !truncated) { - depth += 1; - const byTarget = new Map(); - const locals = new Map(); - const globals = new Set(); - for (const node of frontier) { - if (!byTarget.has(node.fn)) byTarget.set(node.fn, []); - byTarget.get(node.fn).push(node); - if (node.local) { - const k = versionKey(node.crate, node.version); - if (!locals.has(k)) locals.set(k, { name: node.crate, version: node.version, targets: [] }); - locals.get(k).targets.push(node.fn); - } else { - globals.add(node.fn); - } +const CALLERS_LOCAL_NAMES = ` + MATCH (s:Symbol {name: $name}) + UNWIND $versions AS ver + MATCH (cc:Crate {name: $crate})-[:VERSION_OF]->(v:Version {version: ver}) + MATCH (v)-[:HAS_FN]->(f:Fn)-[:CALLS]->(s) + WITH f LIMIT $cap + MATCH (f)-[:IS]->(cs:Symbol) + WITH cs.name AS fn, count(*) AS nv + ORDER BY nv DESC, fn + WITH count(*) AS functionCount, collect({name: fn, versions: nv})[0..$per] AS functions + RETURN $crate AS crate, functionCount, functions + LIMIT 1`; + +const SYMBOL_HOME = ` + MATCH (s:Symbol {name: $name}) + OPTIONAL MATCH (s)-[:IN]->(t:Crate) + RETURN s.name AS name, t.name AS crate, COUNT { (s)<-[:CALLS]-() } AS calls, + COUNT { (s)<-[:IS]-() } AS definitions + LIMIT 1`; + +const cmpVersions = (a, b) => { + const x = parseVersion(a); + const y = parseVersion(b); + if (!x || !y) return a.localeCompare(b); + return cmp(x, y); +}; + +const versionStatus = (targetCrate, versions, row) => { + if (!versions) return 'verified'; + if (targetCrate && normCrate(row.crate) === normCrate(targetCrate)) return versions.includes(row.version) ? 'same' : 'rejected'; + if (!row.reqs.length) return 'unverified'; + for (const req of row.reqs) { + for (const v of versions) { + if (requirementMatches(req, v) !== false) return 'verified'; } + } + return 'rejected'; +}; - const edges = [ - ...(globals.size ? await read(GLOBAL_HOP, { targets: [...globals] }) : []), - ...(locals.size ? await read(LOCAL_HOP, { locals: [...locals.values()] }) : []), - ]; - - const next = []; - for (const e of edges) { - for (const node of byTarget.get(e.target) ?? []) { - let verified = true; - if (node.local) { - if (e.name !== node.crate || e.version !== node.version) continue; - } else { - if (normCrate(e.crate) !== normCrate(node.crate)) continue; - if (node.version !== null) { - if (normCrate(e.name) === normCrate(node.crate)) { - if (e.version !== node.version) continue; - } else if (e.reqs.length === 0) { - verified = false; - } else if (!e.reqs.some((r) => requirementMatches(r, node.version) !== false)) { - rejected += 1; - continue; - } - } - } - - const key = `${versionKey(e.name, e.version)}::${e.callsite}`; - if (seen.has(key)) continue; - seen.add(key); - if (rows.length >= limit) { - truncated = true; - break; - } - if (!verified) unverified += 1; - const path = [e.callsite, ...node.path]; - rows.push({ depth, crate: e.name, version: e.version, function: e.callsite, calls: e.target, verified, path }); - - const sym = calleeCrate(e.callsite); - if (!sym) continue; // core/std generic or no path: nothing records calls into it - const own = normCrate(sym) === normCrate(e.name); - next.push({ fn: e.callsite, crate: e.name, version: e.version, local: !own, path }); - } - if (truncated) break; +/** + * Who calls `name`, grouped by caller crate, without function names. `versions` (optional) pins the + * callee crate to those versions: a caller version of another crate is `verified` when one of its + * DEPENDS_ON requirements on the callee crate admits one of them, `unverified` when it has no such + * dependency (the call came through a generic instantiated elsewhere), and dropped (`rejected`) when + * every requirement excludes them. `local` = `{crate, versions}` restricts the walk to those versions + * (the symbol is a foreign monomorphised copy inside that crate's binary). + */ +const aggregateCallers = async (read, int, { name, versions, local, cap }) => { + const started = Date.now(); + const rows = local + ? await read(CALLERS_LOCAL_AGG, { name, crate: local.crate, versions: local.versions.slice(0, 200), cap: int(cap) }) + : await read(CALLERS_AGG, { name, cap: int(cap) }); + const targetCrate = rows[0]?.targetCrate ?? null; + const total = rows[0]?.total ?? 0; + let scanned = 0; + let rejected = 0; + const rejectedByCrate = new Map(); + const byCrate = new Map(); + for (const row of rows) { + scanned += row.fns; + const status = versionStatus(targetCrate, versions, row); + if (status === 'rejected') { + rejected += 1; + rejectedByCrate.set(row.crate, (rejectedByCrate.get(row.crate) ?? 0) + 1); + continue; } - frontier = next; + if (!byCrate.has(row.crate)) byCrate.set(row.crate, { crate: row.crate, downloads: row.downloads ?? 0, versions: [] }); + byCrate.get(row.crate).versions.push({ version: row.version, latest: row.latest, fns: row.fns, status }); + } + const callers = [...byCrate.values()].map((c) => { + c.versions.sort((a, b) => Number(b.latest) - Number(a.latest) || -cmpVersions(a.version, b.version)); + const unverified = c.versions.filter((v) => v.status === 'unverified').length; + return { ...c, fns: c.versions.reduce((n, v) => n + v.fns, 0), verified: c.versions.length - unverified, unverified }; + }); + callers.sort((a, b) => b.downloads - a.downloads || b.fns - a.fns || a.crate.localeCompare(b.crate)); + return { targetCrate, total, scanned, rejected, rejectedByCrate, callers, names: new Map(), elapsedMs: Date.now() - started }; +}; + +// Small LRU of aggregate promises: concurrent requests for one key share the query, failures are evicted. +const aggregates = new Map(); +const cachedAggregate = (key, compute) => { + const hit = aggregates.get(key); + if (hit && Date.now() - hit.at < CACHE_TTL_MS) { + aggregates.delete(key); + aggregates.set(key, hit); + return { promise: hit.promise, cached: true }; + } + const entry = { at: Date.now(), promise: compute() }; + aggregates.delete(key); + aggregates.set(key, entry); + entry.promise.catch(() => aggregates.get(key) === entry && aggregates.delete(key)); + while (aggregates.size > CACHE_SIZE) aggregates.delete(aggregates.keys().next().value); + return { promise: entry.promise, cached: false }; +}; + +const aggregateFor = (read, int, { name, versions, local, cap }) => { + const key = JSON.stringify([name, versions, local && { crate: local.crate, versions: local.versions.slice(0, 200) }, local ? null : cap]); + return cachedAggregate(key, () => aggregateCallers(read, int, { name, versions, local, cap })); +}; + +const SORT_VALUE = { + downloads: (c) => c.downloads, + functions: (c) => c.fns, + versions: (c) => c.versions.length, + crate: (c) => c.crate, +}; + +const sortCallers = (list, sort, desc) => { + if (sort === 'downloads' && desc) return list; // the aggregate's own order + const val = SORT_VALUE[sort] ?? SORT_VALUE.downloads; + const dir = desc ? -1 : 1; + return [...list].sort((a, b) => { + const x = val(a); + const y = val(b); + const r = typeof x === 'number' ? x - y : String(x).localeCompare(String(y)); + return dir * r || b.downloads - a.downloads || a.crate.localeCompare(b.crate); + }); +}; + +/** Fills `agg.names` for `crates` (page rows not fetched before). */ +const fetchNames = async (read, int, agg, { name, local, cap }, crates) => { + if (!crates.length) return; + const rows = local + ? await read(CALLERS_LOCAL_NAMES, { + name, + crate: local.crate, + versions: crates[0].versions.map((v) => v.version), + cap: int(cap), + per: int(FUNCTIONS_PER_CRATE), + }) + : await read(CALLERS_NAMES, { + name, + cap: int(cap), + crates: crates.map((c) => c.crate), + keys: crates.flatMap((c) => c.versions.map((v) => `${c.crate}@${v.version}`)), + per: int(FUNCTIONS_PER_CRATE), + limit: int(crates.length), + }); + for (const r of rows) agg.names.set(r.crate, { functionCount: r.functionCount, functions: r.functions }); + for (const c of crates) if (!agg.names.has(c.crate)) agg.names.set(c.crate, { functionCount: 0, functions: [] }); +}; + +const intParam = (value, fallback, lo, hi) => { + const n = Number.parseInt(value ?? '', 10); + return Number.isFinite(n) ? Math.min(hi, Math.max(lo, n)) : fallback; +}; + +/** Query-string -> callersPage / callerCrateNames options (shared by both routes). */ +export const parseCallersQuery = (q) => { + const versions = String(q.versions ?? '').trim(); + const localCrate = String(q.localCrate ?? '').trim(); + const sort = CALLER_SORTS.includes(q.sort) ? q.sort : 'downloads'; + return { + name: String(q.fn ?? '').trim(), + versions: versions ? versions.split(',').filter(Boolean) : null, + local: localCrate ? { crate: localCrate, versions: String(q.localVersions ?? '').split(',').filter(Boolean) } : null, + cap: intParam(q.cap, DEFAULT_FN_CAP, 1000, MAX_FN_CAP), + offset: intParam(q.offset, 0, 0, 10_000_000), + limit: intParam(q.limit, DEFAULT_PAGE, 1, MAX_PAGE), + sort, + desc: q.desc === undefined ? sort !== 'crate' : q.desc === '1' || q.desc === 'true', + filter: String(q.filter ?? '').trim(), + q: String(q.q ?? '').trim(), + }; +}; + +/** + * One page of caller crates. Totals cover the whole filtered set; `filter` is a caller crate name + * (exact, case-insensitive) or a `*` / `?` pattern. + */ +export const callersPage = async (read, int, opts) => { + const started = Date.now(); + const { name, versions, local, cap, offset, limit, sort, desc, filter } = opts; + const { promise, cached } = aggregateFor(read, int, opts); + const agg = await promise; + const test = nameTester(filter); + const kept = filter ? agg.callers.filter((c) => test(c.crate)) : agg.callers; + const sorted = sortCallers(kept, sort, desc); + const page = sorted.slice(offset, offset + limit); + await fetchNames(read, int, agg, { name, local, cap }, page.filter((c) => !agg.names.has(c.crate))); + const rejected = filter ? [...agg.rejectedByCrate].reduce((n, [crate, k]) => n + (test(crate) ? k : 0), 0) : agg.rejected; + return { + target: { name, crate: agg.targetCrate, versions, local }, + callers: page.map((c) => ({ ...c, ...agg.names.get(c.crate) })), + meta: { + total: agg.total, + scanned: agg.scanned, + cap, + truncated: !local && agg.total > cap, + rejected, + crates: kept.length, + versions: kept.reduce((n, c) => n + c.versions.length, 0), + fns: kept.reduce((n, c) => n + c.fns, 0), + unverified: kept.reduce((n, c) => n + c.unverified, 0), + allCrates: agg.callers.length, + offset, + limit, + hasMore: offset + page.length < kept.length, + sort, + desc, + filter, + cached, + aggregateMs: agg.elapsedMs, + elapsedMs: Date.now() - started, + }, + }; +}; + +/** + * Caller crate names for the filter box's hints, off the cached aggregate: partial and + * case-insensitive (`-` and `_` alike), prefix matches first, then by downloads. + */ +export const callerCrateNames = async (read, int, opts, { limit = 12 } = {}) => { + const { promise } = aggregateFor(read, int, opts); + const agg = await promise; + const q = normCrate(opts.q.toLowerCase()); + const test = hasWildcard(q) ? nameTester(opts.q) : null; + const scored = []; + for (const c of agg.callers) { + const n = normCrate(c.crate.toLowerCase()); + const at = test ? (test(c.crate) ? 0 : -1) : n.indexOf(q); + if (at < 0) continue; + scored.push({ c, rank: n === q ? 0 : at === 0 ? 1 : 2 }); } + scored.sort((a, b) => a.rank - b.rank || b.c.downloads - a.c.downloads || a.c.crate.localeCompare(b.c.crate)); + return { + names: scored.slice(0, Math.min(50, limit)).map(({ c }) => ({ crate: c.crate, downloads: c.downloads, fns: c.fns })), + matches: scored.length, + }; +}; + +export const symbolHome = (read, name) => read(SYMBOL_HOME, { name }).then((rows) => rows[0] ?? null); + +// --------------------------------------------------------------------------- +// Callees: what one function calls, in the newest version that defines it (or a chosen one). - return { rows, depth, truncated, unverified, rejected, exhausted: !truncated && frontier.length === 0 }; +const CALLEES = ` + MATCH (s:Symbol {name: $name})<-[:IS]-(f:Fn)<-[:HAS_FN]-(v:Version) + WHERE ($crate IS NULL OR v.name = $crate) AND ($version IS NULL OR v.version = $version) + WITH f, v + ORDER BY coalesce(v.latest, false) DESC, v.semver_major DESC, v.semver_minor DESC, v.semver_patch DESC, v.version DESC + LIMIT 1 + MATCH (f)-[c:CALLS]->(cs:Symbol)-[:IN]->(t:Crate) + RETURN v.name AS crate, v.version AS version, t.name AS callee, t.downloads AS downloads, cs.name AS name, + c.copies AS copies, COUNT { (cs)<-[:CALLS]-() } AS calls + ORDER BY callee, calls DESC, name`; + +// Same for a generic function, which is never compiled in its own crate, only monomorphised in +// callers: take the copy in the most-downloaded caller (`from` tells the UI where it came from). +const CALLEES_ANY = ` + MATCH (s:Symbol {name: $name})<-[:IS]-(f:Fn)<-[:HAS_FN]-(v:Version)<-[:VERSION_OF]-(cc:Crate) + WITH f, v + ORDER BY coalesce(cc.downloads, 0) DESC, coalesce(v.latest, false) DESC, v.version DESC + LIMIT 1 + MATCH (f)-[c:CALLS]->(cs:Symbol)-[:IN]->(t:Crate) + RETURN v.name AS crate, v.version AS version, t.name AS callee, t.downloads AS downloads, cs.name AS name, + c.copies AS copies, COUNT { (cs)<-[:CALLS]-() } AS calls + ORDER BY callee, calls DESC, name`; + +export const calleesOf = async (read, { name, crate = null, version = null }) => { + const started = Date.now(); + let rows = await read(CALLEES, { name, crate, version }); + if (!rows.length && version) rows = await read(CALLEES, { name, crate, version: null }); + if (!rows.length) rows = await read(CALLEES_ANY, { name }); + const byCrate = new Map(); + for (const r of rows) { + if (!byCrate.has(r.callee)) byCrate.set(r.callee, { crate: r.callee, downloads: r.downloads ?? 0, functions: [] }); + byCrate.get(r.callee).functions.push({ name: r.name, copies: r.copies ?? 1, calls: r.calls }); + } + const callees = [...byCrate.values()].sort((a, b) => b.functions.length - a.functions.length || a.crate.localeCompare(b.crate)); + return { + from: rows.length ? { crate: rows[0].crate, version: rows[0].version } : null, + callees, + meta: { crates: callees.length, functions: rows.length, elapsedMs: Date.now() - started }, + }; }; diff --git a/render/server/config.mjs b/render/server/config.mjs index 6bd9ada..19adc5d 100644 --- a/render/server/config.mjs +++ b/render/server/config.mjs @@ -23,10 +23,11 @@ export const config = { }, // Queries always run in a read transaction, but you may want this off on a shared deployment. allowCypher: env('RENDER_ALLOW_CYPHER', 'true') !== 'false', - depsMaxEdges: int('RENDER_DEPS_MAX_EDGES', 50_000), - depsDefaultEdges: int('RENDER_DEPS_DEFAULT_EDGES', 5_000), + // Deps view: crates drawn per tree by default, and the most a request may ask for. + depsDefaultNodes: int('RENDER_DEPS_DEFAULT_NODES', 300), + depsMaxNodes: int('RENDER_DEPS_MAX_NODES', 2_000), depsCacheMs: int('RENDER_DEPS_CACHE_MS', 600_000), - // Scans every DEPENDS_ON edge. + // Whole-relationship-type scans (unsafe:all live fallback, ecosystem-wide queries). depsTimeoutMs: int('RENDER_DEPS_TIMEOUT_MS', 180_000), lod: { iterations: int('RENDER_LOD_ITERATIONS', 300), @@ -36,6 +37,9 @@ export const config = { workerHeapMb: int('RENDER_LOD_WORKER_HEAP_MB', 8192), budget: int('RENDER_LOD_BUDGET', 600), maxLevel: int('RENDER_LOD_MAX_LEVEL', 12), + // JSON snapshot of the loaded viewport index, keyed on LodMeta.built_at: a restart reads it in seconds + // instead of streaming every crate and LOD_DEP edge again (minutes). Empty disables it. + cachePath: env('RENDER_LOD_INDEX_CACHE', '/tmp/painter-lod-index.json'), // Positions every N iterations, so a crash can be resumed. checkpointPath: env('RENDER_LOD_CHECKPOINT', '/tmp/painter-lod-checkpoint.json'), }, @@ -63,8 +67,8 @@ export const describeConfig = () => ({ queryTimeoutMs: config.neo4j.queryTimeoutMs, }, allowCypher: config.allowCypher, - depsMaxEdges: config.depsMaxEdges, - depsDefaultEdges: config.depsDefaultEdges, + depsDefaultNodes: config.depsDefaultNodes, + depsMaxNodes: config.depsMaxNodes, lod: { iterations: config.lod.iterations, budget: config.lod.budget, maxLevel: config.lod.maxLevel }, jobsUrl: config.jobsUrl || null, jobs: config.jobs, diff --git a/render/server/deps.mjs b/render/server/deps.mjs new file mode 100644 index 0000000..ae08b44 --- /dev/null +++ b/render/server/deps.mjs @@ -0,0 +1,399 @@ +// Deps view: a bounded crate-level dependency tree around one root. +// +// Forward ("dependencies") walks DEPENDS_ON from the root's latest version (or, in `any` mode, the +// precomputed crate-level LOD_DEP edges the Map build aggregates from every version) one level at a +// time; reverse ("dependents") walks LOD_DEP backwards, since popular crates have 10^5 dependents and +// a DEPENDS_ON expansion over them times out, then verifies each candidate against its latest version +// when the mode asks for it. Every level is one query over the whole frontier (Crate.name is unique +// constrained, VERSION_OF fans out to a crate's versions only), so a tree costs depth + 1 queries. +// +// The walk is bounded by a node budget, not an edge cap: when a level would exceed it, the +// highest-downloaded new crates are kept and each parent remembers how many of its children were +// dropped (`hidden`). Crates on the last level report how many children they would have had +// (`unexpanded`), so the client can show "+N" and fetch them on demand with `expand`. + +import neo4j from 'neo4j-driver'; + +export const KINDS = ['normal', 'build', 'dev']; +const KIND_LABELS = { normal: 'Normal', build: 'Build', dev: 'Dev' }; + +const CHUNK = 250; +/** Candidates per crate for the reverse walk before latest-version verification trims them. */ +const REVERSE_CANDIDATES = 400; +/** + * A reverse level expands crates best-downloaded first while their summed crate-level in-degree stays + * under this; expanding one crate costs time proportional to its in-degree (serde: 130k, 0.4 s), so + * a level of 300 popular crates would otherwise take minutes. The rest keep their `+N` count. + */ +const REVERSE_INDEGREE_BUDGET = 60_000; +/** At most this many crates of a reverse frontier are expanded per level (best-downloaded first). */ +const REVERSE_FRONTIER_MAX = 50; +/** + * Candidates per reverse level across the whole frontier. Each candidate costs a version seek plus a + * DEPENDS_ON check (~1 ms), so a level is bounded to about this many milliseconds. + */ +const REVERSE_CANDIDATES_PER_LEVEL = 1_500; +/** + * Reverse trees keep at most this many new dependents per crate on level 1, 2, ... (best-downloaded + * first); popular crates have 10^4-10^5 dependents and a full budget of them is an unreadable grid. + * The rest stay in the crate's "+N". + */ +const REVERSE_FANOUT = [30, 3]; + +export const parseKinds = (value) => { + const wanted = String(value ?? 'normal') + .split(',') + .map((k) => k.trim().toLowerCase()) + .filter((k) => KINDS.includes(k)); + return wanted.length ? [...new Set(wanted)] : ['normal']; +}; + +const asBool = (value, fallback = false) => { + if (value === undefined || value === null || value === '') return fallback; + return value === '1' || value === 'true' || value === true; +}; + +export const parseDepsQuery = (query, config) => { + const depthRaw = Number.parseInt(query.depth ?? '', 10); + const budgetRaw = Number.parseInt(query.limit ?? '', 10); + return { + mode: query.mode === 'any' ? 'any' : 'latest', + direction: query.direction === 'in' ? 'in' : 'out', + kinds: parseKinds(query.kinds), + optional: asBool(query.optional, false), + depth: Number.isFinite(depthRaw) && depthRaw >= 0 ? Math.min(depthRaw, 12) : 2, + budget: + Number.isFinite(budgetRaw) && budgetRaw > 0 ? Math.min(budgetRaw, config.depsMaxNodes) : config.depsDefaultNodes, + }; +}; + +const chunked = (items, size) => { + const out = []; + for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size)); + return out; +}; + +const kindLabels = (kinds) => kinds.map((k) => KIND_LABELS[k]); + +/** + * Binds `v` to crate `a`'s current version: crates.io's default version (registry job, + * `Crate.default_version`; one seek on the (name, version) constraint) or, when that is unset or its + * node has not been imported yet, the highest version (`Version.latest`, which needs a VERSION_OF + * expansion over every version of the crate, ~1.5 ms per crate). + */ +const CURRENT_VERSION = ` + CALL { + WITH a + MATCH (v:Version {name: a.name, version: a.default_version}) RETURN v + UNION + WITH a + WITH a WHERE a.default_version IS NULL OR NOT EXISTS { (:Version {name: a.name, version: a.default_version}) } + MATCH (a)-[:VERSION_OF]->(v:Version {latest: true}) RETURN v + }`; + +/** + * One forward level: for every crate in `names`, its direct dependencies. Rows are aggregated per + * (from, to) pair: `kinds` is the set of dependency kinds, `optional` is true only when every + * declaration is optional, `req` is one of the requirements. + */ +const forwardLevel = async (readRows, names, opts) => { + const rows = []; + for (const chunk of chunked(names, CHUNK)) { + const part = + opts.mode === 'latest' + ? await readRows( + `MATCH (a:Crate) WHERE a.name IN $names + ${CURRENT_VERSION} + MATCH (v)-[d:DEPENDS_ON]->(b:Crate) + WHERE d.kind IN $kinds AND ($optional OR coalesce(d.optional, false) = false) + WITH a, b, collect(DISTINCT d.kind) AS kinds, + min(CASE WHEN coalesce(d.optional, false) THEN 1 ELSE 0 END) = 1 AS optional, + collect(DISTINCT d.requirement) AS reqs + RETURN a.name AS from, b.name AS to, kinds, optional, reqs[0] AS req, null AS versions, + b.downloads AS downloads, COUNT { (b)<-[:LOD_DEP]-() } AS dependents`, + { names: chunk, kinds: kindLabels(opts.kinds), optional: opts.optional }, + ) + : await readRows( + `MATCH (a:Crate) WHERE a.name IN $names + MATCH (a)-[r:LOD_DEP]->(b:Crate) + RETURN a.name AS from, b.name AS to, [] AS kinds, false AS optional, null AS req, r.versions AS versions, + b.downloads AS downloads, COUNT { (b)<-[:LOD_DEP]-() } AS dependents`, + { names: chunk }, + ); + rows.push(...part); + } + return rows; +}; + +/** + * One reverse level: for every crate in `names`, the crates that depend on it, best-downloaded first. + * LOD_DEP folds every version, so in `latest` mode each candidate is checked against its latest + * version and dropped when that version no longer declares the dependency (or only with an excluded + * kind). `dependents` is the candidate's own in-degree so node sizes mean the same thing both ways. + */ +const reverseLevel = async (readRows, names, opts, perNode) => { + const rows = []; + for (const chunk of chunked(names, CHUNK)) { + const part = + opts.mode === 'latest' + ? await readRows( + `MATCH (c:Crate) WHERE c.name IN $names + MATCH (c)<-[r:LOD_DEP]-(a:Crate) + WITH c, a, r ORDER BY coalesce(a.downloads, 0) DESC + WITH c, collect({a: a, versions: r.versions})[0..$perNode] AS top + UNWIND top AS t + WITH c, t.a AS a, t.versions AS versions + ${CURRENT_VERSION} + MATCH (v)-[d:DEPENDS_ON]->(c) + WHERE d.kind IN $kinds AND ($optional OR coalesce(d.optional, false) = false) + WITH c, a, versions, collect(DISTINCT d.kind) AS kinds, + min(CASE WHEN coalesce(d.optional, false) THEN 1 ELSE 0 END) = 1 AS optional, + collect(DISTINCT d.requirement) AS reqs + RETURN a.name AS from, c.name AS to, kinds, optional, reqs[0] AS req, versions, + a.downloads AS downloads, COUNT { (a)<-[:LOD_DEP]-() } AS dependents`, + { names: chunk, perNode: neo4j.int(perNode), kinds: kindLabels(opts.kinds), optional: opts.optional }, + ) + : await readRows( + `MATCH (c:Crate) WHERE c.name IN $names + MATCH (c)<-[r:LOD_DEP]-(a:Crate) + WITH c, a, r ORDER BY coalesce(a.downloads, 0) DESC + WITH c, collect({a: a, versions: r.versions})[0..$perNode] AS top + UNWIND top AS t + WITH c, t.a AS a, t.versions AS versions + RETURN a.name AS from, c.name AS to, [] AS kinds, false AS optional, null AS req, versions, + a.downloads AS downloads, COUNT { (a)<-[:LOD_DEP]-() } AS dependents`, + { names: chunk, perNode: neo4j.int(perNode) }, + ); + rows.push(...part); + } + return rows; +}; + +/** The crate on the far side of an edge row, as seen from the walk's frontier. */ +const other = (row, opts) => (opts.direction === 'out' ? row.to : row.from); +const self = (row, opts) => (opts.direction === 'out' ? row.from : row.to); + +const level = (readRows, names, opts, perNode) => + opts.direction === 'out' ? forwardLevel(readRows, names, opts) : reverseLevel(readRows, names, opts, perNode); + +const rootInfo = async (readRows, name) => { + const rows = await readRows( + `MATCH (a:Crate {name: $name}) + ${CURRENT_VERSION} + WITH a, v LIMIT 1 + OPTIONAL MATCH (v)-[d:DEPENDS_ON]->() + WITH a, v, d.kind AS kind, coalesce(d.optional, false) AS optional, count(d) AS n + RETURN a.name AS name, a.downloads AS downloads, COUNT { (a)<-[:LOD_DEP]-() } AS dependents, a.description AS description, + v.version AS latest, [x IN collect({kind: toLower(kind), optional: optional, n: n}) WHERE x.n > 0] AS declared`, + { name }, + ); + return rows[0] ?? null; +}; + +const edgeOf = (row) => ({ + from: row.from, + to: row.to, + kinds: (row.kinds ?? []).map((k) => String(k).toLowerCase()), + optional: row.optional === true, + req: row.req ?? null, + versions: row.versions ?? null, +}); + +/** Reverse walk: the part of a frontier worth expanding within REVERSE_INDEGREE_BUDGET (see above). */ +const reverseFrontier = (frontier, nodes) => { + const ranked = frontier + .map((name) => nodes.get(name)) + .sort((a, b) => (b.downloads ?? 0) - (a.downloads ?? 0) || a.name.localeCompare(b.name)); + const kept = []; + let sum = 0; + for (const n of ranked) { + if (n.dependents === 0) continue; + if (kept.length > 0 && sum + n.dependents > REVERSE_INDEGREE_BUDGET) continue; + sum += n.dependents; + kept.push(n.name); + if (kept.length >= REVERSE_FRONTIER_MAX) break; + } + return kept; +}; + +/** + * Reverse walks only fetch the best-downloaded candidates per crate, but a crate's total crate-level + * in-degree is known exactly (`dependents`), so "+N" is every dependent not drawn rather than the + * candidates that happened to be fetched. This also covers the last level and crates the frontier + * budget skipped. In `latest` mode it is an upper bound: some of those crates only depended on it in + * older versions. + */ +const fixReverseHidden = (nodes, edges, opts) => { + if (opts.direction !== 'in') return; + const shown = new Map(); + for (const e of edges) shown.set(e.to, (shown.get(e.to) ?? 0) + 1); + for (const n of nodes.values()) { + n.hidden = Math.max(0, n.dependents - (shown.get(n.name) ?? 0)); + n.unexpanded = 0; + } +}; + +/** + * Breadth-first tree from `root`. Returns `{root, nodes, edges, meta}` where every node has + * `depth` (hops from the root along kept edges), `downloads`, `dependents` (crate-level in-degree), + * `hidden` (children dropped by the budget) and `unexpanded` (children beyond max depth). + */ +export const depsTree = async (readRows, root, opts) => { + const started = Date.now(); + const info = await rootInfo(readRows, root); + if (!info) return null; + + const nodes = new Map([[root, { name: root, depth: 0, downloads: info.downloads ?? null, dependents: info.dependents ?? 0, hidden: 0, unexpanded: 0 }]]); + const edges = []; + const edgeKeys = new Set(); + let frontier = [root]; + let depth = 0; + let truncated = false; + let queries = 1; + const levels = []; + + const addEdge = (row) => { + const key = `${row.from}\u0000${row.to}`; + if (edgeKeys.has(key)) return; + edgeKeys.add(key); + edges.push(edgeOf(row)); + }; + + while (frontier.length > 0 && depth < opts.depth) { + if (opts.direction === 'in' && depth > 0) frontier = reverseFrontier(frontier, nodes); + if (frontier.length === 0) break; + // Only `room` new crates can be kept, so fetching many more candidates per crate is wasted work. + const room = opts.budget - nodes.size; + if (room <= 0) { + truncated = true; + break; + } + const levelStarted = Date.now(); + const perNode = Math.min(REVERSE_CANDIDATES, Math.max(20, Math.ceil(room * 1.5)), Math.max(10, Math.ceil(REVERSE_CANDIDATES_PER_LEVEL / frontier.length))); + let rows = await level(readRows, frontier, opts, perNode); + queries += 1; + depth += 1; + levels.push({ + depth, + frontier: frontier.length, + indegree: opts.direction === 'in' ? frontier.reduce((sum, n) => sum + nodes.get(n).dependents, 0) : undefined, + rows: rows.length, + elapsedMs: Date.now() - levelStarted, + }); + if (opts.direction === 'in') { + const cap = REVERSE_FANOUT[Math.min(depth - 1, REVERSE_FANOUT.length - 1)]; + const perParent = new Map(); + const sorted = [...rows].sort((a, b) => (b.downloads ?? 0) - (a.downloads ?? 0)); + rows = []; + for (const row of sorted) { + if (!nodes.has(other(row, opts))) { + const count = perParent.get(self(row, opts)) ?? 0; + if (count >= cap) { + truncated = true; + continue; + } + perParent.set(self(row, opts), count + 1); + } + rows.push(row); + } + } + // New crates this level would introduce, best-downloaded first (ties: most depended-on, then name). + const fresh = new Map(); + for (const row of rows) { + const name = other(row, opts); + if (nodes.has(name) || fresh.has(name)) continue; + fresh.set(name, { name, depth, downloads: row.downloads ?? null, dependents: row.dependents ?? 0, hidden: 0, unexpanded: 0 }); + } + let kept = fresh; + if (fresh.size > room) { + truncated = true; + const ranked = [...fresh.values()].sort( + (a, b) => (b.downloads ?? 0) - (a.downloads ?? 0) || b.dependents - a.dependents || a.name.localeCompare(b.name), + ); + kept = new Map(ranked.slice(0, Math.max(room, 0)).map((n) => [n.name, n])); + } + for (const n of kept.values()) nodes.set(n.name, n); + for (const row of rows) { + const name = other(row, opts); + if (nodes.has(name)) addEdge(row); + else nodes.get(self(row, opts)).hidden += 1; + } + frontier = [...kept.keys()]; + if (frontier.length === 0) break; + } + + // Children of the last level walked (max depth or a full budget) that were not fetched, so the client can offer to expand them. A reverse + // walk knows them already: every node's `dependents` is its exact crate-level in-degree. + if (opts.direction === 'out' && frontier.length > 0) { + const rows = await level(readRows, frontier, opts, Math.min(REVERSE_CANDIDATES, opts.budget)); + queries += 1; + for (const row of rows) { + const name = other(row, opts); + if (!nodes.has(name)) nodes.get(self(row, opts)).unexpanded += 1; + else addEdge(row); + } + } + + fixReverseHidden(nodes, edges, opts); + + return { + root, + rootInfo: { latest: info.latest ?? null, description: info.description ?? null, declared: info.declared ?? [] }, + nodes: [...nodes.values()], + edges, + meta: { + ...opts, + depthReached: depth, + nodes: nodes.size, + edges: edges.length, + hidden: [...nodes.values()].reduce((s, n) => s + n.hidden, 0), + unexpanded: [...nodes.values()].reduce((s, n) => s + n.unexpanded, 0), + truncated, + queries, + levels, + elapsedMs: Date.now() - started, + }, + }; +}; + +/** + * One crate's direct neighbours in the walk's direction, every one of them up to `budget` + * (best-downloaded first), for progressive expansion of a tree the client already holds. + */ +export const depsExpand = async (readRows, name, opts) => { + const started = Date.now(); + const info = await rootInfo(readRows, name); + if (!info) return null; + const rows = await level(readRows, [name], opts, opts.budget); + const ranked = rows + .map((row) => ({ row, name: other(row, opts) })) + .sort((a, b) => (b.row.downloads ?? 0) - (a.row.downloads ?? 0) || (b.row.dependents ?? 0) - (a.row.dependents ?? 0) || a.name.localeCompare(b.name)); + const kept = ranked.slice(0, opts.budget); + const nodes = new Map( + kept.map(({ row, name: n }) => [n, { name: n, depth: 1, downloads: row.downloads ?? null, dependents: row.dependents ?? 0, hidden: 0, unexpanded: 0 }]), + ); + nodes.set(name, { name, depth: 0, downloads: info.downloads ?? null, dependents: info.dependents ?? 0, hidden: Math.max(0, rows.length - kept.length), unexpanded: 0 }); + const edges = kept.map(({ row }) => edgeOf(row)); + // How many children each new crate has in turn, so the client can offer to expand them too. + if (kept.length > 0 && opts.direction === 'out') { + const edgeKeys = new Set(edges.map((e) => `${e.from}\u0000${e.to}`)); + const next = await level(readRows, [...nodes.keys()].filter((n) => n !== name), opts, Math.min(REVERSE_CANDIDATES, opts.budget)); + for (const row of next) { + const target = other(row, opts); + const key = `${row.from}\u0000${row.to}`; + if (nodes.has(target)) { + if (!edgeKeys.has(key)) { + edgeKeys.add(key); + edges.push(edgeOf(row)); + } + } else nodes.get(self(row, opts)).unexpanded += 1; + } + } + fixReverseHidden(nodes, edges, opts); + return { + root: name, + nodes: [...nodes.values()], + edges, + meta: { ...opts, total: rows.length, truncated: rows.length > kept.length, elapsedMs: Date.now() - started }, + }; +}; diff --git a/render/server/index.mjs b/render/server/index.mjs index 70fc394..c3712bd 100644 --- a/render/server/index.mjs +++ b/render/server/index.mjs @@ -5,7 +5,9 @@ import { fileURLToPath } from 'node:url'; import { existsSync } from 'node:fs'; import { config, describeConfig } from './config.mjs'; import { collectGraph } from './graph.mjs'; -import { findCallers, normCrate } from './callers.mjs'; +import { calleesOf, callerCrateNames, callersPage, parseCallersQuery, resolveSymbol, searchSymbols, symbolHome } from './callers.mjs'; +import { crateOverview } from './overview.mjs'; +import { hasWildcard, nameTester, wildcardRegex } from './match.mjs'; import { SORTS as UNSAFE_SORTS, unsafeCrate, unsafeDeps, unsafeStats, unsafeTop } from './unsafe.mjs'; import { BUILD_SORTS, @@ -43,6 +45,7 @@ import { import { createLodJobs } from './lod/jobs.mjs'; import { LodIndex } from './lod/index.mjs'; import { createDriver, waitForNeo4j } from './db.mjs'; +import { depsExpand, depsTree, parseDepsQuery } from './deps.mjs'; import { STATS_IDS, overviewStats, readStats, readStatsAges } from './stats.mjs'; const here = path.dirname(fileURLToPath(import.meta.url)); @@ -149,30 +152,49 @@ api.get( api.post('/stats/refresh', forwardToJobs('POST', '/api/stats/refresh')); -// A TEXT index on Crate.name (docker/queries.md) makes CONTAINS fast; without it this is a label scan. -const searchCrates = async (q, limit) => - readGraph( +// Suggestions are always partial: CONTAINS on the TEXT index (docker/queries.md), exact and prefix hits first. +// A query with `*` / `?` is a pattern instead, ranked by downloads. +const searchCrates = async (input, limit) => { + const text = String(input ?? '').trim(); + if (!text) return []; + if (hasWildcard(text)) { + return readGraph( + `MATCH (c:Crate) WHERE c.name =~ $re + RETURN c.name AS name ORDER BY coalesce(c.downloads, 0) DESC, c.name LIMIT $limit`, + { re: wildcardRegex(text), limit: neo4j.int(limit) }, + ).then((g) => g.rows.map((r) => r.name)); + } + return readGraph( `MATCH (c:Crate) WHERE toLower(c.name) CONTAINS $q WITH c, toLower(c.name) AS lname RETURN c.name AS name ORDER BY lname = $q DESC, lname STARTS WITH $q DESC, size(c.name), c.name LIMIT $limit`, - { q, limit: neo4j.int(limit) }, + { q: text.toLowerCase(), limit: neo4j.int(limit) }, ).then((g) => g.rows.map((r) => r.name)); +}; +// Plain input must name a crate exactly (case-insensitive); only a wildcard query picks the best +// (most-downloaded) match. The indexed lookup covers the typed and lowercase spellings; the +// case-insensitive scan only runs for names like `Inflector` typed in another case. const resolveCrateName = async (input) => { const q = String(input ?? '').trim(); if (!q) return null; - const exact = await readGraph(`MATCH (c:Crate {name: $name}) RETURN c.name AS name LIMIT 1`, { name: q }); + if (hasWildcard(q)) return (await searchCrates(q, 1))[0] ?? null; + const exact = await readGraph(`MATCH (c:Crate) WHERE c.name IN $names RETURN c.name AS name LIMIT 1`, { names: [...new Set([q, q.toLowerCase()])] }); if (exact.rows.length) return exact.rows[0].name; - const matches = await searchCrates(q.toLowerCase(), 1); - return matches[0] ?? null; + const folded = await readGraph(`MATCH (c:Crate) WHERE toLower(c.name) = $q RETURN c.name AS name LIMIT 1`, { q: q.toLowerCase() }); + return folded.rows[0]?.name ?? null; }; +/** 404 text for a name that did not resolve, pointing at the wildcard syntax. */ +const noCrate = (q) => + hasWildcard(q) ? `no crate matches the pattern '${q}'` : `no crate named '${q}' (pick a suggestion, or use * for a pattern, e.g. ${q}*)`; + api.get( '/crates/search', wrap(async (req, res) => { - const q = String(req.query.q ?? '').trim().toLowerCase(); + const q = String(req.query.q ?? '').trim(); const limit = clampLimit(req.query.limit, 20); if (!q) return res.json({ names: [] }); res.json({ names: await searchCrates(q, limit) }); @@ -194,7 +216,7 @@ api.get( const requested = String(req.params.name); const name = await resolveCrateName(requested); if (!name) { - return res.status(404).json({ error: `no crate matches '${requested}'` }); + return res.status(404).json({ error: noCrate(requested) }); } const allVersions = req.query.versions === 'all'; const deps = req.query.deps !== 'false'; @@ -258,6 +280,28 @@ api.get( }), ); +// Default view: see server/overview.mjs. `parts` = deps,versions,calls,dependents; `version` + `from=version` +// hangs the edges off that version node; `skip` pages versions / dependents; `root=1` marks the layout root. +api.get( + '/graph/overview/:name', + wrap(async (req, res) => { + const requested = String(req.params.name); + const name = await resolveCrateName(requested); + if (!name) return res.status(404).json({ error: noCrate(requested) }); + const g = await crateOverview(readGraph, neo4j.int, { + name, + parts: req.query.parts, + limit: req.query.limit, + skip: req.query.skip, + version: req.query.version, + from: req.query.from, + root: req.query.root === '1' || req.query.root === 'true', + }); + if (!g) return res.status(404).json({ error: `crate '${name}' not found` }); + res.json(g); + }), +); + api.post( '/graph/expand', wrap(async (req, res) => { @@ -304,247 +348,92 @@ api.post( }), ); -const aggregatedDependsOn = (fromId, toId, versions) => ({ - id: `dep:${fromId}->${toId}`, - type: 'DEPENDS_ON', - from: fromId, - to: toId, - properties: { versions, aggregated: true }, -}); - -const depsFromRows = (g) => { - for (const row of g.rows) { - if (row.a && row.b) g.rels.push(aggregatedDependsOn(row.a.id, row.b.id, row.versions)); - } - return g; -}; - -const depsCache = new Map(); - -// Informational only: computed in the background so it never blocks a response. -let depsTotalPairs = { value: null, at: 0, inFlight: null }; - -const totalCratePairs = () => { - const stale = Date.now() - depsTotalPairs.at > config.depsCacheMs; - if (stale && !depsTotalPairs.inFlight) { - depsTotalPairs.inFlight = readGraph( - `MATCH (v:Version)-[:DEPENDS_ON]->(b:Crate) - RETURN count(DISTINCT [v.name, b.name]) AS pairs`, - {}, - { timeoutMs: config.depsTimeoutMs }, - ) - .then((g) => { - const pairs = g.rows[0]?.pairs ?? null; - if (pairs !== null && pairs > 0) depsTotalPairs = { ...depsTotalPairs, value: pairs, at: Date.now() }; - }) - .catch((err) => log(`total crate pairs count failed: ${err.message}`)) - .finally(() => { - depsTotalPairs.inFlight = null; - }); - } - return depsTotalPairs.value; -}; +// Deps view (server/deps.mjs): a bounded tree around one crate. `direction=out` walks dependencies of the +// latest version (`mode=latest`, `kinds=normal,build,dev`, `optional=1` to include optional ones) or of any +// version (`mode=any`, the Map's LOD_DEP edges); `direction=in` walks dependents. `limit` is a node budget. +const readRowsDeps = (cypher, params) => readGraph(cypher, params, { timeoutMs: config.depsTimeoutMs }).then((g) => g.rows); -const clampDepsLimit = (value) => { - const n = Number.parseInt(value ?? '', 10); - if (!Number.isFinite(n) || n <= 0) return config.depsDefaultEdges; - return Math.min(n, config.depsMaxEdges); -}; +api.get( + '/deps/tree', + wrap(async (req, res) => { + const input = String(req.query.crate ?? '').trim(); + if (!input) return res.status(400).json({ error: 'crate is required' }); + const name = await resolveCrateName(input); + if (!name) return res.status(404).json({ error: noCrate(input) }); + const tree = await depsTree(readRowsDeps, name, parseDepsQuery(req.query, config)); + if (!tree) return res.status(404).json({ error: `no crate named '${name}'` }); + res.json({ ...tree, meta: { ...tree.meta, resolvedName: name, query: input } }); + }), +); -// Without `crate`, return the most-used edges (by version count) so a capped view still shows the ecosystem core. api.get( - '/graph/deps', + '/deps/expand', wrap(async (req, res) => { - const limit = clampDepsLimit(req.query.limit); - const crateInput = String(req.query.crate ?? '').trim(); + const name = String(req.query.crate ?? '').trim(); + if (!name) return res.status(400).json({ error: 'crate is required' }); + const part = await depsExpand(readRowsDeps, name, parseDepsQuery(req.query, config)); + if (!part) return res.status(404).json({ error: `no crate named '${name}'` }); + res.json(part); + }), +); - if (!crateInput) { - const cached = depsCache.get(limit); - if (cached && Date.now() - cached.at < config.depsCacheMs) { - return res.json({ ...cached.payload, meta: { ...cached.payload.meta, cached: true } }); - } - // Version nodes carry the crate name, so skip the VERSION_OF hop and resolve the source Crate only - // for the `limit` winners. Scans every DEPENDS_ON edge once, hence the long timeout. - const g = depsFromRows( - await readGraph( - `MATCH (v:Version)-[:DEPENDS_ON]->(b:Crate) - WITH v.name AS aName, b, count(*) AS versions - ORDER BY versions DESC - LIMIT $limit - MATCH (a:Crate {name: aName}) - RETURN a, b, versions`, - { limit: neo4j.int(limit) }, - { timeoutMs: config.depsTimeoutMs }, - ), - ); - const totalPairs = totalCratePairs(); - const payload = { - ...stripRows(g), - meta: { - ...g.meta, - limit, - totalPairs, - truncated: g.rels.length >= limit && (totalPairs === null || totalPairs > limit), - }, - }; - if (g.rels.length > 0) depsCache.set(limit, { at: Date.now(), payload }); - return res.json(payload); - } +// Callers view (server/callers.mjs): one search box, then callers aggregated per crate. - const name = await resolveCrateName(crateInput); - if (!name) return res.status(404).json({ error: `no crate matches '${crateInput}'` }); - const direction = req.query.direction === 'out' || req.query.direction === 'in' ? req.query.direction : 'both'; - const cap = neo4j.int(limit); - - const g = await readGraph(`MATCH (c:Crate {name: $name}) RETURN c`, { name }); - if (direction !== 'in') { - mergeInto( - g, - depsFromRows( - await readGraph( - `MATCH (a:Crate {name: $name}) - MATCH (v:Version {name: $name})-[:DEPENDS_ON]->(b:Crate) - WITH a, b, count(*) AS versions - ORDER BY versions DESC LIMIT $limit - RETURN a, b, versions`, - { name, limit: cap }, - ), - ), - ); - } - if (direction !== 'out') { - mergeInto( - g, - depsFromRows( - await readGraph( - `MATCH (b:Crate {name: $name})<-[:DEPENDS_ON]-(v:Version) - WITH b, v.name AS aName, count(*) AS versions - ORDER BY versions DESC LIMIT $limit - MATCH (a:Crate {name: aName}) - RETURN a, b, versions`, - { name, limit: cap }, - ), - ), - ); - } - res.json({ ...stripRows(g), meta: { ...g.meta, limit, resolvedName: name } }); +api.get( + '/symbols/search', + wrap(async (req, res) => { + res.json(await searchSymbols(readRowsPlain, neo4j.int, String(req.query.q ?? ''), { limit: clampLimit(req.query.limit, 30) })); }), ); api.get( - '/graph/deps/chain', + '/symbols/resolve', wrap(async (req, res) => { - const q = String(req.query.q ?? '').trim(); - if (!q) return res.status(400).json({ error: 'q is required' }); - const limit = clampDepsLimit(req.query.limit); - const maxDepth = Math.max(0, Number.parseInt(req.query.depth ?? '0', 10) || 0); - const forceExact = req.query.exact === '1' || req.query.exact === 'true'; - const started = Date.now(); - - let rootsGraph = await readGraph(`MATCH (c:Crate {name: $q}) RETURN c`, { q }); - let exact = rootsGraph.nodes.length > 0; - let partial = false; - if (!exact && !forceExact && q.length >= 3) { - partial = true; - rootsGraph = await readGraph( - `MATCH (c:Crate) WHERE toLower(c.name) CONTAINS $q RETURN c ORDER BY size(c.name), c.name`, - { q: q.toLowerCase() }, - { timeoutMs: config.depsTimeoutMs }, - ); - } - if (rootsGraph.nodes.length === 0) { - return res.status(404).json({ - error: partial ? `no crate name contains '${q}'` : q.length >= 3 ? `no crate named '${q}'` : `no crate named '${q}' (3+ characters enables partial matching)`, - }); - } + res.json(await resolveSymbol(readRowsPlain, neo4j.int, String(req.query.q ?? ''))); + }), +); - const g = { nodes: [...rootsGraph.nodes], rels: [], meta: { records: rootsGraph.meta.records, elapsedMs: 0 } }; - const roots = rootsGraph.nodes.map((n) => n.properties.name); - const visited = new Set(roots); - let frontier = roots; - let depth = 0; - let truncated = false; - - while (frontier.length > 0 && (maxDepth === 0 || depth < maxDepth)) { - const remaining = limit - g.rels.length; - if (remaining <= 0) { - truncated = true; - break; - } - // Crate.name is unique-constrained (indexed). - const level = depsFromRows( - await readGraph( - `MATCH (a:Crate) WHERE a.name IN $names - MATCH (a)-[:VERSION_OF]-(v:Version)-[:DEPENDS_ON]->(b:Crate) - WITH a, b, count(v) AS versions - ORDER BY a.name = $q DESC, versions DESC - RETURN a, b, versions - LIMIT $limit`, - { names: frontier, q, limit: neo4j.int(remaining + 1) }, - { timeoutMs: config.depsTimeoutMs }, - ), - ); - if (level.rels.length > remaining) { - truncated = true; - level.rels = level.rels.slice(0, remaining); - const keep = new Set(level.rels.flatMap((r) => [r.from, r.to])); - level.nodes = level.nodes.filter((n) => keep.has(n.id)); - } - mergeInto(g, level); - depth += 1; - const next = []; - for (const n of level.nodes) { - const name = n.properties.name; - if (!visited.has(name)) { - visited.add(name); - next.push(name); - } - } - frontier = next; - if (truncated) break; +// Paged: `offset`, `limit` (<= 500), `sort`, `desc`, `filter` (caller crate name or pattern) over an +// aggregate cached per (fn, versions, local, cap); function names only for the page's crates. +api.get( + '/symbols/callers', + wrap(async (req, res) => { + const opts = parseCallersQuery(req.query); + if (!opts.name) return res.status(400).json({ error: 'fn is required' }); + const result = await callersPage(readRowsPlain, neo4j.int, opts); + if (!result.target.crate && !result.meta.allCrates) { + const home = await symbolHome(readRowsPlain, opts.name); + if (!home) return res.status(404).json({ error: `no function named '${opts.name}' in the call graph (calls are only recorded from crate versions painter has exported)` }); + result.target.crate = home.crate; + result.meta.total = home.calls; + result.meta.definitions = home.definitions; } - - res.json({ - ...stripRows(g), - meta: { - ...g.meta, - elapsedMs: Date.now() - started, - query: q, - partial, - exact, - roots, - depth, - maxDepth, - limit, - truncated, - crates: g.nodes.length, - edges: g.rels.length, - }, - }); + res.json(result); }), ); -// Symbol paths spell `my-crate` as `my_crate`; accept either. -const crateSpellings = (name) => [...new Set([name, normCrate(name), name.replace(/_/g, '-')])]; - -// Names ending in `q` rank first so `spawn` finds `tokio::task::spawn::spawn`. -const searchFunctions = async (crate, q, limit) => - readGraph( - `MATCH (t:Crate) WHERE t.name IN $names - MATCH (t)<-[:IN]-(s:Symbol) - WHERE $q = '' OR toLower(s.name) CONTAINS $q - WITH s.name AS name, COUNT { (s)<-[:CALLS]-() } AS calls - ORDER BY toLower(name) ENDS WITH $q DESC, calls DESC, size(name) - RETURN name, calls LIMIT $limit`, - { names: crateSpellings(crate), q: q.toLowerCase(), limit: neo4j.int(limit) }, - ).then((g) => g.rows); +// Caller crate names for the Callers filter hints (`q` partial), off the same cached aggregate. +api.get( + '/symbols/callers/crates', + wrap(async (req, res) => { + const opts = parseCallersQuery(req.query); + if (!opts.name) return res.status(400).json({ error: 'fn is required' }); + res.json(await callerCrateNames(readRowsPlain, neo4j.int, opts, { limit: clampLimit(req.query.limit, 12) })); + }), +); api.get( - '/functions/search', + '/symbols/callees', wrap(async (req, res) => { - const crate = String(req.query.crate ?? '').trim(); - if (!crate) return res.status(400).json({ error: 'crate is required' }); - res.json({ functions: await searchFunctions(crate, String(req.query.q ?? '').trim(), clampLimit(req.query.limit, 20)) }); + const name = String(req.query.fn ?? '').trim(); + if (!name) return res.status(400).json({ error: 'fn is required' }); + res.json( + await calleesOf(readRowsPlain, { + name, + crate: String(req.query.crate ?? '').trim() || null, + version: String(req.query.version ?? '').trim() || null, + }), + ); }), ); @@ -561,56 +450,6 @@ api.get( }), ); -api.get( - '/callers', - wrap(async (req, res) => { - const crate = String(req.query.crate ?? '').trim(); - const fn = String(req.query.fn ?? '').trim(); - if (!crate || !fn) return res.status(400).json({ error: 'crate and fn are required' }); - const version = String(req.query.version ?? '').trim() || null; - const exact = req.query.match === 'exact'; - const maxDepth = Math.min(10, Math.max(1, Number.parseInt(req.query.depth ?? '3', 10) || 3)); - const limit = clampLimit(req.query.limit, config.neo4j.maxRows); - const started = Date.now(); - - if (version) { - const found = await readGraph(`MATCH (c:Crate)-[:VERSION_OF]-(v:Version {version: $version}) WHERE c.name IN $names RETURN v LIMIT 1`, { - names: crateSpellings(crate), - version, - }); - if (!found.nodes.length) return res.status(404).json({ error: `no version ${version} of crate '${crate}'` }); - } - - const targets = exact ? [fn] : (await searchFunctions(crate, fn, 500)).map((f) => f.name); - if (!targets.length) { - return res.status(404).json({ - error: `nothing calls a function of '${crate}' containing '${fn}' (calls are only recorded from crate versions painter has exported)`, - }); - } - - const read = (cypher, params) => readGraph(cypher, params).then((g) => g.rows); - const result = await findCallers(read, { crate, version, targets, maxDepth, limit }); - res.json({ - rows: result.rows, - meta: { - crate, - version, - fn, - exact, - targets, - maxDepth, - depth: result.depth, - limit, - truncated: result.truncated, - exhausted: result.exhausted, - unverified: result.unverified, - rejected: result.rejected, - elapsedMs: Date.now() - started, - }, - }); - }), -); - let lodIndex = null; let lodLoading = null; let lodLoadError = null; @@ -620,7 +459,7 @@ const loadLodIndex = () => { lodLoadError = null; const started = Date.now(); log('lod index: loading'); - lodLoading = LodIndex.load(driver, config.neo4j.database, { timeoutMs: config.lod.timeoutMs, maxLevel: config.lod.maxLevel }) + lodLoading = LodIndex.load(driver, config.neo4j.database, { timeoutMs: config.lod.timeoutMs, cachePath: config.lod.cachePath || null, log }) .then((idx) => { lodIndex = idx; log(idx ? `lod index: ${idx.count.toLocaleString()} crates, ${idx.edgeCount.toLocaleString()} edges in ${Date.now() - started} ms` : 'lod index: no build in database'); @@ -1035,7 +874,8 @@ api.get( } const budget = Math.max(10, Math.min(Number.parseInt(req.query.budget ?? '', 10) || config.lod.budget, 5000)); const minCell = Math.max(0, Number.parseFloat(req.query.minCell ?? '') || 0); - res.json(idx.viewport(bbox, { budget, minCell })); + const maxRels = Math.max(0, Math.min(Number.parseInt(req.query.maxRels ?? '', 10) || budget, 5000)); + res.json(idx.viewport(bbox, { budget, maxRels, minCell })); }), ); @@ -1046,11 +886,34 @@ api.get( if (!idx) return; const name = String(req.query.name ?? '').trim(); const hit = idx.locate(name); - if (!hit) return res.status(404).json({ error: `crate '${name}' is not on the map` }); + if (!hit) return res.status(404).json({ error: hasWildcard(name) ? `no crate on the map matches '${name}'` : `no crate named '${name}' on the map (pick a suggestion, or use * for a pattern, e.g. ${name}*)` }); res.json(hit); }), ); +// The crate's most important direct dependencies and dependents, from the in-memory index (no database query). +api.get( + '/lod/neighbours', + wrap(async (req, res) => { + const idx = await requireLod(res); + if (!idx) return; + const name = String(req.query.name ?? '').trim(); + const hit = idx.neighbours(name, clampLimit(req.query.limit, 30)); + if (!hit) return res.status(404).json({ error: hasWildcard(name) ? `no crate on the map matches '${name}'` : `no crate named '${name}' on the map (pick a suggestion, or use * for a pattern, e.g. ${name}*)` }); + res.json(hit); + }), +); + +// Largest communities with centroid, hull, and dominant crates; computed once when the index loads. +api.get( + '/lod/communities', + wrap(async (_req, res) => { + const idx = await requireLod(res); + if (!idx) return; + res.json({ communities: idx.communities, total: idx.communityCount }); + }), +); + api.get( '/lod/search', wrap(async (req, res) => { diff --git a/render/server/lod/index.mjs b/render/server/lod/index.mjs index 9bab012..9880f8c 100644 --- a/render/server/lod/index.mjs +++ b/render/server/lod/index.mjs @@ -1,30 +1,47 @@ -// Nothing is precomputed per level: a viewport costs one pass over the in-view points per level tried plus one over all edges -// (tens of ms on a full graph), which is cheaper than holding per-level edge tables. +// In-memory viewport index for the Map. Nothing is precomputed per zoom level: a viewport is one +// walk down the global importance order (downloads + dependents) picking crates inside the bbox +// that keep a minimum on-screen spacing, so a crate that is a landmark at one zoom stays one at every +// deeper zoom. A sparse density grid over the same bbox shows where the other crates are without +// sending them. Community summaries (summary.mjs) are built once at load. +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import path from 'node:path'; +import neo4j from 'neo4j-driver'; import { loadLodPositions, streamRead } from './neo4j-io.mjs'; +import { summarizeCommunities } from './summary.mjs'; +import { hasWildcard, nameTester } from '../match.mjs'; -const packCell = (level, ix, iy) => `${level}:${ix}:${iy}`; +const SNAPSHOT_VERSION = 2; +/** Density grid columns across the requested bbox; rows follow its aspect ratio. */ +const DENSITY_COLS = 96; +const VIEW_CACHE_MAX = 64; +/** Communities that get their own colour (must match MAP_PALETTE.length in src/mapOverlay.ts); density cells are tinted by majority among them. */ +const PALETTE_SLOTS = 16; +const NEIGHBOUR_CACHE_MAX = 256; + +const quantile = (sorted, q) => sorted[Math.min(sorted.length - 1, Math.max(0, Math.floor(q * (sorted.length - 1))))]; + +/** Importance: downloads dominate, dependents (LOD_DEP in-degree) break ties and rank crates the registry has not seen. */ +const importance = (downloads, dependents) => Math.log10(downloads + 1) + 1.5 * Math.log10(dependents + 1); export class LodIndex { - constructor({ positions, edges, meta, maxLevel }) { - const n = positions.length; + /** + * Columnar input: `names` plus parallel `x, y, community, degree, downloads` arrays and edge + * index triples `ea -> eb` with weight `ev` (versions declaring the dependency). + */ + constructor({ names, x, y, community, degree, downloads, ea, eb, ev, meta }) { + const n = names.length; this.count = n; this.meta = meta; - this.maxLevel = maxLevel; - this.names = new Array(n); + this.names = names; this.byName = new Map(); - this.x = new Float64Array(n); - this.y = new Float64Array(n); - this.community = new Int32Array(n); - this.degree = new Int32Array(n); - positions.forEach((p, i) => { - this.names[i] = p.name; - this.byName.set(p.name, i); - this.x[i] = p.x; - this.y[i] = p.y; - this.community[i] = p.community; - this.degree[i] = p.degree; - }); + for (let i = 0; i < n; i++) this.byName.set(names[i], i); + this.x = Float64Array.from(x); + this.y = Float64Array.from(y); + this.community = Int32Array.from(community); + this.degree = Int32Array.from(degree); + this.downloads = Float64Array.from(downloads); // Bounds with a small margin so edge points are strictly inside. let minX = Infinity; let minY = Infinity; @@ -39,45 +56,138 @@ export class LodIndex { const pad = Math.max(maxX - minX, maxY - minY, 1) * 0.01; this.bounds = { minX: minX - pad, minY: minY - pad, maxX: maxX + pad, maxY: maxY + pad }; this.span = Math.max(this.bounds.maxX - this.bounds.minX, this.bounds.maxY - this.bounds.minY); - this.nx = new Float32Array(n); - this.ny = new Float32Array(n); - for (let i = 0; i < n; i++) { - this.nx[i] = (this.x[i] - this.bounds.minX) / this.span; - this.ny[i] = (this.y[i] - this.bounds.minY) / this.span; - } - const m = edges.length; - this.ea = new Int32Array(m); - this.eb = new Int32Array(m); - this.ev = new Float32Array(m); + // Isolated crates sit on an outer ring, so the whole bounds are mostly empty: the 1.5-98.5 + // percentile box is where the ecosystem actually is and is what the UI opens on. + const xs = Float64Array.from(this.x).sort(); + const ys = Float64Array.from(this.y).sort(); + this.core = { minX: quantile(xs, 0.015), minY: quantile(ys, 0.015), maxX: quantile(xs, 0.985), maxY: quantile(ys, 0.985) }; + + // Edges as CSR both ways (deps = out, dependents = in). + const m = ea.length; + const valid = new Uint8Array(m); + this.outdeg = new Int32Array(n); + this.indeg = new Int32Array(n); let k = 0; - for (const [a, b, v] of edges) { - const ia = this.byName.get(a); - const ib = this.byName.get(b); - if (ia === undefined || ib === undefined || ia === ib) continue; - this.ea[k] = ia; - this.eb[k] = ib; - this.ev[k] = v; + for (let e = 0; e < m; e++) { + const a = ea[e]; + const b = eb[e]; + if (a === undefined || b === undefined || a < 0 || b < 0 || a >= n || b >= n || a === b) continue; + valid[e] = 1; + this.outdeg[a] += 1; + this.indeg[b] += 1; k += 1; } this.edgeCount = k; + this.outStart = new Int32Array(n + 1); + this.inStart = new Int32Array(n + 1); + for (let i = 0; i < n; i++) { + this.outStart[i + 1] = this.outStart[i] + this.outdeg[i]; + this.inStart[i + 1] = this.inStart[i] + this.indeg[i]; + } + this.outIdx = new Int32Array(k); + this.outW = new Float32Array(k); + this.inIdx = new Int32Array(k); + this.inW = new Float32Array(k); + const outFill = Int32Array.from(this.outStart.subarray(0, n)); + const inFill = Int32Array.from(this.inStart.subarray(0, n)); + for (let e = 0; e < m; e++) { + if (!valid[e]) continue; + const a = ea[e]; + const b = eb[e]; + const w = ev[e]; + this.outIdx[outFill[a]] = b; + this.outW[outFill[a]] = w; + outFill[a] += 1; + this.inIdx[inFill[b]] = a; + this.inW[inFill[b]] = w; + inFill[b] += 1; + } + + this.score = new Float32Array(n); + for (let i = 0; i < n; i++) this.score[i] = importance(this.downloads[i], this.indeg[i]); + const order = new Int32Array(n); + for (let i = 0; i < n; i++) order[i] = i; + order.sort((a, b) => this.score[b] - this.score[a] || this.indeg[b] - this.indeg[a] || a - b); + this.order = order; + this.rank = new Int32Array(n); + for (let r = 0; r < n; r++) this.rank[order[r]] = r; + + const summary = summarizeCommunities(this, { limit: 48 }); + this.communities = summary.communities; + this.communityCount = summary.total; + this.communityById = new Map(this.communities.map((c) => [c.id, c])); + // Palette slot per crate: 0..PALETTE_SLOTS-1 for the top communities, PALETTE_SLOTS for everything else. + const slotOf = new Map(this.communities.slice(0, PALETTE_SLOTS).map((c, k) => [c.id, k])); + this.slot = new Uint8Array(n); + for (let i = 0; i < n; i++) this.slot[i] = slotOf.get(this.community[i]) ?? PALETTE_SLOTS; + this.slotCommunity = this.communities.slice(0, PALETTE_SLOTS).map((c) => c.id); + this.viewCache = new Map(); + this.neighbourCache = new Map(); this.loadedAt = new Date().toISOString(); } - static async load(driver, database, { timeoutMs, maxLevel }) { - const { positions, meta } = await loadLodPositions(driver, database, { timeoutMs }); - // LodMeta is written last, so positions without meta means a build is mid-write: report not ready. - if (positions.length === 0 || !meta || !meta.built_at) return null; - const edges = []; + /** + * Loads from the database, or from the JSON snapshot at `cachePath` when its `built_at` matches + * the database's LodMeta (a full load streams 340k crates and 2.9M edges, ~4 min against a remote + * database; the snapshot takes seconds). The snapshot is rewritten after every database load. + */ + static async load(driver, database, { timeoutMs, cachePath = null, log = () => {} } = {}) { + const meta = await loadLodMeta(driver, database); + // LodMeta is written last, so no meta means a build is mid-write (or none exists): report not ready. + if (!meta || !meta.built_at) return null; + const builtAt = String(meta.built_at); + if (cachePath) { + try { + const snap = await readSnapshot(cachePath); + if (snap && snap.version === SNAPSHOT_VERSION && snap.built_at === builtAt) { + log(`lod index: using snapshot ${cachePath} (built ${builtAt})`); + return new LodIndex({ ...snap, meta }); + } + if (snap) log(`lod index: snapshot is for build ${snap.built_at}, database has ${builtAt}; reloading`); + } catch (err) { + log(`lod index: snapshot unreadable (${err.message}); loading from the database`); + } + } + const { positions } = await loadLodPositions(driver, database, { timeoutMs }); + if (positions.length === 0) return null; + const byName = new Map(); + const names = new Array(positions.length); + const x = new Float64Array(positions.length); + const y = new Float64Array(positions.length); + const community = new Int32Array(positions.length); + const degree = new Int32Array(positions.length); + const downloads = new Float64Array(positions.length); + positions.forEach((p, i) => { + names[i] = p.name; + byName.set(p.name, i); + x[i] = p.x; + y[i] = p.y; + community[i] = p.community; + degree[i] = p.degree; + downloads[i] = p.downloads; + }); + const ea = []; + const eb = []; + const ev = []; await streamRead(driver, database, `MATCH (a:Crate)-[r:LOD_DEP]->(b:Crate) RETURN a.name AS a, b.name AS b, r.versions AS n`, {}, { timeoutMs, - onRecord: (r) => edges.push([r.get('a'), r.get('b'), Number(r.get('n'))]), + onRecord: (r) => { + const a = byName.get(r.get('a')); + const b = byName.get(r.get('b')); + if (a === undefined || b === undefined) return; + ea.push(a); + eb.push(b); + const v = r.get('n'); + ev.push(neo4j.isInt(v) ? v.toNumber() : Number(v)); + }, }); - return new LodIndex({ positions, edges, meta, maxLevel }); - } - - cellOf(i, level) { - const g = 1 << level; - return packCell(level, Math.min(g - 1, Math.floor(this.nx[i] * g)), Math.min(g - 1, Math.floor(this.ny[i] * g))); + const columns = { names, x: Array.from(x), y: Array.from(y), community: Array.from(community), degree: Array.from(degree), downloads: Array.from(downloads), ea, eb, ev }; + if (cachePath) { + writeSnapshot(cachePath, { version: SNAPSHOT_VERSION, built_at: builtAt, ...columns }) + .then(() => log(`lod index: snapshot written to ${cachePath}`)) + .catch((err) => log(`lod index: snapshot write failed: ${err.message}`)); + } + return new LodIndex({ ...columns, meta }); } inView(x0, y0, x1, y1) { @@ -88,212 +198,268 @@ export class LodIndex { return out; } - // `minCell` is derived from the client's zoom so clusters keep a minimum on-screen spacing. + /** + * `budget` landmarks for the bbox, at least `minCell` layout units apart (the client derives it from + * its zoom, so spacing is constant on screen), plus their edges and a density grid of everything else. + */ viewport(bbox, { budget, maxRels = budget, minCell = 0 }) { const started = Date.now(); const x0 = Math.min(bbox.x0, bbox.x1); const x1 = Math.max(bbox.x0, bbox.x1); const y0 = Math.min(bbox.y0, bbox.y1); const y1 = Math.max(bbox.y0, bbox.y1); - // The whole-map view is the most common and most expensive query; memoise it per budget. - const whole = x0 <= this.bounds.minX && y0 <= this.bounds.minY && x1 >= this.bounds.maxX && y1 >= this.bounds.maxY; - const levelCap = minCell > 0 ? Math.max(0, Math.min(this.maxLevel, Math.floor(Math.log2(this.span / minCell)))) : this.maxLevel; - const cacheKey = `${budget}:${maxRels}:${levelCap}`; - if (whole) { - this.wholeCache ??= new Map(); - const hit = this.wholeCache.get(cacheKey); - if (hit) return { ...hit, meta: { ...hit.meta, elapsedMs: 0, cached: true } }; + const key = [x0, y0, x1, y1, minCell].map((v) => v.toPrecision(4)).join(',') + `:${budget}:${maxRels}`; + const hit = this.viewCache.get(key); + if (hit) { + this.viewCache.delete(key); + this.viewCache.set(key, hit); + return { ...hit, meta: { ...hit.meta, elapsedMs: Date.now() - started, cached: true } }; } - const result = this.viewportUncached({ x0, y0, x1, y1 }, { budget, maxRels, started, levelCap }); - if (whole) this.wholeCache.set(cacheKey, result); + const result = this.viewportUncached({ x0, y0, x1, y1 }, { budget, maxRels, minCell, started }); + this.viewCache.set(key, result); + if (this.viewCache.size > VIEW_CACHE_MAX) this.viewCache.delete(this.viewCache.keys().next().value); return result; } - viewportUncached({ x0, y0, x1, y1 }, { budget, maxRels, started, levelCap }) { - const idx = this.inView(x0, y0, x1, y1); - - if (idx.length <= budget && levelCap >= this.maxLevel) { - return this.cratesPayload(idx, { maxRels, started, inView: idx.length, level: this.maxLevel }); + viewportUncached({ x0, y0, x1, y1 }, { budget, maxRels, minCell, started }) { + const w = Math.max(x1 - x0, 1e-9); + const h = Math.max(y1 - y0, 1e-9); + // Density grid and in-view count: one pass over every point. + const cols = DENSITY_COLS; + const rows = Math.max(8, Math.min(160, Math.round((cols * h) / w))); + const counts = new Int32Array(cols * rows); + // Per-cell tally per palette slot: the majority coloured community tints the cell, grey if "other" wins. + const slots = PALETTE_SLOTS + 1; + const tally = new Int32Array(cols * rows * slots); + let inView = 0; + for (let i = 0; i < this.count; i++) { + const px = this.x[i]; + const py = this.y[i]; + if (px < x0 || px > x1 || py < y0 || py > y1) continue; + inView += 1; + const cx = Math.min(cols - 1, Math.floor(((px - x0) / w) * cols)); + const cy = Math.min(rows - 1, Math.floor(((py - y0) / h) * rows)); + const c = cy * cols + cx; + counts[c] += 1; + tally[c * slots + this.slot[i]] += 1; } - - // Count distinct cells per level coarse to fine, bailing out once over budget; only the winning level is materialised. - let level = 0; - for (let l = 1; l <= levelCap; l++) { - const g = 1 << l; - const seen = new Set(); - let over = false; - for (const i of idx) { - const ix = Math.min(g - 1, Math.floor(this.nx[i] * g)); - const iy = Math.min(g - 1, Math.floor(this.ny[i] * g)); - seen.add(ix * g + iy); - if (seen.size > budget) { - over = true; - break; + const cells = []; + for (let c = 0; c < counts.length; c++) { + if (counts[c] === 0) continue; + let best = PALETTE_SLOTS; + let bestN = tally[c * slots + PALETTE_SLOTS]; + for (let k = 0; k < PALETTE_SLOTS; k++) { + if (tally[c * slots + k] > bestN) { + bestN = tally[c * slots + k]; + best = k; } } - if (over) break; - level = l; + cells.push(c, counts[c], best < PALETTE_SLOTS ? this.slotCommunity[best] : -1); } - const cells = new Map(); - for (const i of idx) { - const key = this.cellOf(i, level); - let c = cells.get(key); - if (!c) { - c = { key, members: [] }; - cells.set(key, c); + + // Landmarks: walk the global importance order, keep those in the bbox that do not crowd a + // higher-ranked pick. Spacing uses a dense occupancy grid of `minCell` cells over the bbox (one + // occupant per cell, 3x3 neighbourhood distance check), so the scan is a few ms even when the + // screen fills up long before the budget and every in-view crate has to be considered. + const picks = []; + const cell = minCell > 0 ? minCell : 0; + const gw = cell ? Math.ceil(w / cell) + 1 : 0; + const gh = cell ? Math.ceil(h / cell) + 1 : 0; + const occupant = cell ? new Int32Array(gw * gh).fill(-1) : null; + const capacity = cell ? gw * gh : Infinity; + let scanned = 0; + for (let r = 0; r < this.count && picks.length < budget && picks.length < capacity; r++) { + const i = this.order[r]; + const px = this.x[i]; + const py = this.y[i]; + if (px < x0 || px > x1 || py < y0 || py > y1) continue; + scanned += 1; + if (occupant) { + const gx = Math.floor((px - x0) / cell); + const gy = Math.floor((py - y0) / cell); + let crowded = occupant[gy * gw + gx] >= 0; + for (let dy = -1; dy <= 1 && !crowded; dy++) { + const yy = gy + dy; + if (yy < 0 || yy >= gh) continue; + for (let dx = -1; dx <= 1; dx++) { + const xx = gx + dx; + if (xx < 0 || xx >= gw) continue; + const j = occupant[yy * gw + xx]; + if (j >= 0 && Math.hypot(this.x[j] - px, this.y[j] - py) < cell) { + crowded = true; + break; + } + } + } + if (crowded) continue; + occupant[gy * gw + gx] = i; } - c.members.push(i); + picks.push(i); } - return this.cellsPayload(cells, { level, maxRels, started, inView: idx.length }); - } - - crateNode(i) { - return { - id: `crate:${this.names[i]}`, - labels: ['Crate'], - properties: { name: this.names[i], x: this.x[i], y: this.y[i], community: this.community[i], degree: this.degree[i] }, - }; - } - cratesPayload(idx, { maxRels, started, inView, level }) { - const visible = new Uint8Array(this.count); - for (const i of idx) visible[i] = 1; - const nodes = idx.map((i) => this.crateNode(i)); + // Edges among the landmarks (out-degree is small, so this is cheap even for serde). + const pickPos = new Int32Array(this.count).fill(-1); + picks.forEach((i, p) => (pickPos[i] = p)); const rels = []; - for (let k = 0; k < this.edgeCount; k++) { - const a = this.ea[k]; - const b = this.eb[k]; - if (visible[a] && visible[b]) { - rels.push({ - id: `dep:crate:${this.names[a]}->crate:${this.names[b]}`, - type: 'DEPENDS_ON', - from: `crate:${this.names[a]}`, - to: `crate:${this.names[b]}`, - properties: { versions: this.ev[k], aggregated: true }, - }); + for (const a of picks) { + for (let e = this.outStart[a]; e < this.outStart[a + 1]; e++) { + const b = this.outIdx[e]; + if (pickPos[b] < 0) continue; + rels.push({ a, b, versions: this.outW[e] }); } } - rels.sort((p, q) => q.properties.versions - p.properties.versions); - const truncated = rels.length > maxRels; + rels.sort((p, q) => q.versions - p.versions); + const relsTotal = rels.length; + // More than `maxRels` edges is a smear, not information: send none and let the client show the + // selection's own edges instead. + const truncated = relsTotal > maxRels; + const relsOut = (truncated ? [] : rels).map((e) => ({ + id: `dep:crate:${this.names[e.a]}->crate:${this.names[e.b]}`, + type: 'DEPENDS_ON', + from: `crate:${this.names[e.a]}`, + to: `crate:${this.names[e.b]}`, + properties: { versions: e.versions, aggregated: true }, + })); + const level = Math.max(0, Math.round(Math.log2(this.span / Math.max(w, h)))); return { - mode: 'crates', + mode: 'landmarks', level, - nodes, - rels: truncated ? rels.slice(0, maxRels) : rels, - meta: { inView, total: this.count, elapsedMs: Date.now() - started, relsTruncated: truncated, bounds: this.bounds }, + nodes: picks.map((i) => this.crateNode(i)), + rels: relsOut, + density: { x0, y0, x1, y1, cols, rows, cells }, + meta: { + inView, + total: this.count, + shown: picks.length, + candidates: scanned, + relsTotal, + elapsedMs: Date.now() - started, + relsTruncated: truncated, + bounds: this.bounds, + }, }; } - cellsPayload(cells, { level, maxRels, started, inView }) { - const nodes = []; - const elemOf = new Int32Array(this.count).fill(-1); - const ids = []; - const g = 1 << level; - for (const c of cells.values()) { - if (c.members.length === 1) { - const i = c.members[0]; - elemOf[i] = ids.length; - ids.push(`crate:${this.names[i]}`); - nodes.push(this.crateNode(i)); - continue; - } - let sx = 0; - let sy = 0; - const comm = new Map(); - const top = []; - for (const i of c.members) { - sx += this.x[i]; - sy += this.y[i]; - comm.set(this.community[i], (comm.get(this.community[i]) ?? 0) + 1); - elemOf[i] = ids.length; - if (top.length < 3 || this.degree[i] > top[top.length - 1].d) { - top.push({ n: this.names[i], d: this.degree[i] }); - top.sort((p, q) => q.d - p.d); - if (top.length > 3) top.length = 3; - } - } - let community = -1; - let best = -1; - for (const [k, v] of comm) { - if (v > best) { - best = v; - community = k; - } - } - const [, ix, iy] = c.key.split(':').map(Number); - const id = `cell:${c.key}`; - ids.push(id); - nodes.push({ - id, - labels: ['Cluster'], - properties: { - count: c.members.length, - x: sx / c.members.length, - y: sy / c.members.length, - community, - top: top.map((t) => t.n), - level, - bbox: { - x0: this.bounds.minX + (ix / g) * this.span, - y0: this.bounds.minY + (iy / g) * this.span, - x1: this.bounds.minX + ((ix + 1) / g) * this.span, - y1: this.bounds.minY + ((iy + 1) / g) * this.span, - }, - }, - }); + crateNode(i, extra = null) { + return { + id: `crate:${this.names[i]}`, + labels: ['Crate'], + properties: { + name: this.names[i], + x: Math.round(this.x[i] * 10) / 10, + y: Math.round(this.y[i] * 10) / 10, + community: this.community[i], + downloads: this.downloads[i], + dependents: this.indeg[i], + dependencies: this.outdeg[i], + rank: this.rank[i], + ...(extra ?? {}), + }, + }; + } + + /** Index of a crate by exact name, case-insensitively; a `*` / `?` pattern resolves to its most important match. */ + resolve(name) { + const text = String(name ?? '').trim(); + if (!text) return undefined; + if (hasWildcard(text)) { + const test = nameTester(text); + let best; + for (let i = 0; i < this.count; i++) if ((best === undefined || this.rank[i] < this.rank[best]) && test(this.names[i])) best = i; + return best; } - const n = ids.length; - const agg = new Map(); - for (let k = 0; k < this.edgeCount; k++) { - const fa = elemOf[this.ea[k]]; - const fb = elemOf[this.eb[k]]; - if (fa < 0 || fb < 0 || fa === fb) continue; - const key = fa * n + fb; - let e = agg.get(key); - if (!e) { - e = { fa, fb, count: 0, versions: 0 }; - agg.set(key, e); + const i = this.byName.get(text); + if (i !== undefined) return i; + if (!this.byLower) { + this.byLower = new Map(); + for (let j = 0; j < this.count; j++) { + const l = this.names[j].toLowerCase(); + if (!this.byLower.has(l)) this.byLower.set(l, j); } - e.count += 1; - e.versions += this.ev[k]; } - let rels = [...agg.values()]; - rels.sort((p, q) => q.count - p.count); - const truncated = rels.length > maxRels; - if (truncated) rels = rels.slice(0, maxRels); - const relsOut = rels.map((e) => ({ - id: `dep:${ids[e.fa]}->${ids[e.fb]}`, - type: 'DEPENDS_ON', - from: ids[e.fa], - to: ids[e.fb], - properties: { count: e.count, versions: e.versions, aggregated: true }, - })); - return { - mode: 'cells', - level, - nodes, - rels: relsOut, - meta: { inView, total: this.count, cells: cells.size, elapsedMs: Date.now() - started, relsTruncated: truncated, bounds: this.bounds }, - }; + return this.byLower.get(text.toLowerCase()); } locate(name) { + const i = this.resolve(name); + if (i === undefined) return null; + return this.crateNode(i).properties; + } + + /** The crate plus its most important direct dependencies and dependents (crate level, all versions folded). */ + neighbours(name, limit = 30) { const i = this.byName.get(name); if (i === undefined) return null; - return { name, x: this.x[i], y: this.y[i], community: this.community[i], degree: this.degree[i] }; + const key = `${name}:${limit}`; + const hit = this.neighbourCache.get(key); + if (hit) return hit; + const side = (start, idxs, ws) => { + const list = []; + for (let e = start[i]; e < start[i + 1]; e++) list.push(e); + list.sort((p, q) => this.score[idxs[q]] - this.score[idxs[p]]); + return { total: list.length, top: list.slice(0, limit).map((e) => this.crateNode(idxs[e], { versions: ws[e] })) }; + }; + const result = { + crate: this.crateNode(i), + community: this.communityById.get(this.community[i]) ? this.communityLabel(this.community[i]) : null, + dependencies: side(this.outStart, this.outIdx, this.outW), + dependents: side(this.inStart, this.inIdx, this.inW), + }; + this.neighbourCache.set(key, result); + if (this.neighbourCache.size > NEIGHBOUR_CACHE_MAX) this.neighbourCache.delete(this.neighbourCache.keys().next().value); + return result; } + communityLabel(id) { + const c = this.communityById.get(id); + return c ? { id: c.id, size: c.size, top: c.top } : null; + } + + /** Suggestions: partial matches (exact name first), or a `*` / `?` pattern; both by importance. */ search(q, limit = 20) { const needle = q.toLowerCase(); + const test = hasWildcard(q) ? nameTester(q) : (name) => name.toLowerCase().includes(needle); const hits = []; for (let i = 0; i < this.count; i++) { - if (this.names[i].toLowerCase().includes(needle)) hits.push(i); + if (test(this.names[i])) hits.push(i); } - hits.sort((a, b) => (this.names[a] === needle ? -1 : this.names[b] === needle ? 1 : this.degree[b] - this.degree[a])); + const exact = (i) => this.names[i].toLowerCase() === needle; + hits.sort((a, b) => (exact(a) ? -1 : exact(b) ? 1 : this.rank[a] - this.rank[b])); return hits.slice(0, limit).map((i) => this.names[i]); } status() { - return { loadedAt: this.loadedAt, crates: this.count, edges: this.edgeCount, bounds: this.bounds, maxLevel: this.maxLevel, meta: this.meta }; + return { + loadedAt: this.loadedAt, + crates: this.count, + edges: this.edgeCount, + bounds: this.bounds, + core: this.core, + communities: this.communityCount, + meta: this.meta, + }; } } + +const loadLodMeta = async (driver, database) => { + const session = driver.session({ database, defaultAccessMode: neo4j.session.READ }); + try { + const result = await session.executeRead((tx) => tx.run(`MATCH (m:LodMeta {id: 'default'}) RETURN properties(m) AS m`), { timeout: 60_000 }); + const m = result.records[0]?.get('m') ?? null; + return m ? Object.fromEntries(Object.entries(m).map(([k, v]) => [k, neo4j.isInt(v) ? v.toNumber() : v])) : null; + } finally { + await session.close(); + } +}; + +const readSnapshot = async (file) => { + if (!fs.existsSync(file)) return null; + return JSON.parse(await fsp.readFile(file, 'utf8')); +}; + +const writeSnapshot = async (file, data) => { + await fsp.mkdir(path.dirname(file), { recursive: true }); + const tmp = `${file}.${process.pid}.tmp`; + await fsp.writeFile(tmp, JSON.stringify(data)); + await fsp.rename(tmp, file); +}; diff --git a/render/server/lod/neo4j-io.mjs b/render/server/lod/neo4j-io.mjs index 6fefb7e..cc752af 100644 --- a/render/server/lod/neo4j-io.mjs +++ b/render/server/lod/neo4j-io.mjs @@ -70,7 +70,7 @@ export const loadLodPositions = async (driver, database, { timeoutMs }) => { driver, database, `MATCH (c:Crate) WHERE c.lod_x IS NOT NULL - RETURN c.name AS name, c.lod_x AS x, c.lod_y AS y, c.lod_community AS community, c.lod_degree AS degree`, + RETURN c.name AS name, c.lod_x AS x, c.lod_y AS y, c.lod_community AS community, c.lod_degree AS degree, c.downloads AS downloads`, {}, { timeoutMs, @@ -81,6 +81,8 @@ export const loadLodPositions = async (driver, database, { timeoutMs }) => { y: toNum(r.get('y')), community: toNum(r.get('community')) ?? -1, degree: toNum(r.get('degree')) ?? 0, + // Registry job property; absent until it has run, then used to rank landmarks. + downloads: toNum(r.get('downloads')) ?? 0, }), }, ); diff --git a/render/server/lod/summary.mjs b/render/server/lod/summary.mjs new file mode 100644 index 0000000..c26cb17 --- /dev/null +++ b/render/server/lod/summary.mjs @@ -0,0 +1,110 @@ +// Per-community summaries computed once from the in-memory index (no database query per request): +// where each large community sits on the map, how big it is, and which crates dominate it, so the UI +// can label blobs ("serde · serde_json · thiserror") and give the top communities meaningful colours. + +/** Andrew's monotone chain; points are [x, y]. Returns the hull counter-clockwise without the repeated first point. */ +export const convexHull = (points) => { + if (points.length < 3) return points.slice(); + const pts = points.slice().sort((a, b) => a[0] - b[0] || a[1] - b[1]); + const cross = (o, a, b) => (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]); + const lower = []; + for (const p of pts) { + while (lower.length >= 2 && cross(lower[lower.length - 2], lower[lower.length - 1], p) <= 0) lower.pop(); + lower.push(p); + } + const upper = []; + for (let i = pts.length - 1; i >= 0; i--) { + const p = pts[i]; + while (upper.length >= 2 && cross(upper[upper.length - 2], upper[upper.length - 1], p) <= 0) upper.pop(); + upper.push(p); + } + lower.pop(); + upper.pop(); + return lower.concat(upper); +}; + +/** Keep at most `max` hull vertices, evenly spaced, so payloads stay small. */ +const thinHull = (hull, max = 40) => { + if (hull.length <= max) return hull; + const out = []; + for (let k = 0; k < max; k++) out.push(hull[Math.floor((k * hull.length) / max)]); + return out; +}; + +const quantile = (sorted, q) => sorted[Math.min(sorted.length - 1, Math.max(0, Math.floor(q * (sorted.length - 1))))]; + +/** + * Rank communities by total downloads (summed importance as the tie-break, and the only signal + * before the registry job has run; families of auto-generated crates would otherwise outrank real + * ecosystems by sheer count) and describe the top `limit`: + * centroid, 80th-percentile radius (outliers excluded), a convex hull of the crates within that + * radius, and the dominant crates. ForceAtlas2 keeps Louvain communities only loosely together, so + * `spread` (radius / map span) tells the client how coherent a blob really is. + */ +export const summarizeCommunities = (idx, { limit = 48, topNames = 4 } = {}) => { + const groups = new Map(); + for (let i = 0; i < idx.count; i++) { + const c = idx.community[i]; + if (c < 0) continue; + let g = groups.get(c); + if (!g) { + g = { id: c, members: [], sumScore: 0, downloads: 0 }; + groups.set(c, g); + } + g.members.push(i); + g.sumScore += idx.score[i]; + g.downloads += idx.downloads[i]; + } + const ranked = [...groups.values()].sort((a, b) => b.downloads - a.downloads || b.sumScore - a.sumScore); + const out = []; + for (const g of ranked.slice(0, limit)) { + const n = g.members.length; + let sx = 0; + let sy = 0; + for (const i of g.members) { + sx += idx.x[i]; + sy += idx.y[i]; + } + const cx = sx / n; + const cy = sy / n; + const dist = new Float64Array(n); + for (let k = 0; k < n; k++) { + const i = g.members[k]; + dist[k] = Math.hypot(idx.x[i] - cx, idx.y[i] - cy); + } + const sorted = Float64Array.from(dist).sort(); + const radius = quantile(sorted, 0.8); + const inner = []; + let ix0 = Infinity; + let iy0 = Infinity; + let ix1 = -Infinity; + let iy1 = -Infinity; + for (let k = 0; k < n; k++) { + if (dist[k] > radius) continue; + const i = g.members[k]; + inner.push([idx.x[i], idx.y[i]]); + if (idx.x[i] < ix0) ix0 = idx.x[i]; + if (idx.y[i] < iy0) iy0 = idx.y[i]; + if (idx.x[i] > ix1) ix1 = idx.x[i]; + if (idx.y[i] > iy1) iy1 = idx.y[i]; + } + const top = g.members + .slice() + .sort((a, b) => idx.score[b] - idx.score[a]) + .slice(0, topNames) + .map((i) => idx.names[i]); + out.push({ + id: g.id, + size: n, + downloads: g.downloads, + x: cx, + y: cy, + radius, + spread: radius / idx.span, + bbox: { x0: ix0, y0: iy0, x1: ix1, y1: iy1 }, + hull: thinHull(convexHull(inner)).map(([x, y]) => [Math.round(x), Math.round(y)]), + top, + }); + } + return { communities: out, total: groups.size }; +}; diff --git a/render/server/match.mjs b/render/server/match.mjs new file mode 100644 index 0000000..8191693 --- /dev/null +++ b/render/server/match.mjs @@ -0,0 +1,74 @@ +// Name matching shared by every search box and table filter. +// +// Plain text is an exact, case-insensitive name. Only `*` (any run of characters) and `?` (one +// character) make a query fuzzy: `tokio*`, `*-sys`, `*serde*`, `serde_?son`. Suggestion dropdowns are +// the exception: they always offer partial matches so the user can pick an exact name. + +export const hasWildcard = (q) => /[*?]/.test(String(q ?? '')); + +const escapeRe = (c) => (/[\\^$.|+()[\]{}]/.test(c) ? `\\${c}` : c); +const toRegexBody = (q) => [...q].map((c) => (c === '*' ? '.*' : c === '?' ? '.' : escapeRe(c))).join(''); + +/** Cypher `=~` pattern (full-string, case-insensitive) for a wildcard query. */ +export const wildcardRegex = (q) => `(?i)${toRegexBody(String(q).trim())}`; + +/** JS predicate on a name: exact (case-insensitive) without wildcards, the pattern with them. */ +export const nameTester = (q) => { + const text = String(q ?? '').trim(); + if (!text) return () => true; + if (!hasWildcard(text)) { + const lower = text.toLowerCase(); + return (name) => String(name).toLowerCase() === lower; + } + const re = new RegExp(`^${toRegexBody(text)}$`, 'i'); + return (name) => re.test(String(name)); +}; + +/** + * Longest literal run of a wildcard query (`*tokio-*util` -> `tokio-`), so a pattern search can seed + * from an index-backed CONTAINS before the regex filters it. Empty when the query is all wildcards. + */ +export const longestLiteral = (q) => + String(q) + .split(/[*?]+/) + .reduce((best, part) => (part.length > best.length ? part : best), ''); + +/** + * Cypher filter on `expr` driven by the `$q` / `$re` parameters from `nameParams`: empty matches + * everything, `$re` null is an exact case-insensitive comparison, otherwise the wildcard regex. + */ +export const nameFilter = (expr) => `($q = '' OR ($re IS NULL AND toLower(${expr}) = $q) OR ($re IS NOT NULL AND ${expr} =~ $re))`; + +export const nameParams = (q) => { + const text = String(q ?? '').trim(); + return { q: text.toLowerCase(), re: hasWildcard(text) ? wildcardRegex(text) : null }; +}; + +/** Above this many crates a pattern filters the version scan instead of anchoring on the names. */ +const NAME_ANCHOR_MAX = 5000; + +/** + * Crate-name filter for the version tables (unsafe, build, native, diffs). Their scans walk a sort + * index until the page is full, so a filter that matches few versions (an exact name, a rare pattern) + * would read every version first. Resolve the crate names up front instead (one index lookup, or a + * regex over the ~340k crate names) and start from those crates. + * Returns `{ match, filter, params, crates }` to splice into `MATCH (v:Version) WHERE ... AND filter`. + */ +export const crateNameScope = async (read, int, q) => { + const text = String(q ?? '').trim(); + const all = { match: 'MATCH (v:Version)', filter: 'true', params: {}, crates: null }; + if (!text) return all; + let names; + if (hasWildcard(text)) { + const rows = await read(`MATCH (c:Crate) WHERE c.name =~ $re RETURN c.name AS name LIMIT $max`, { re: wildcardRegex(text), max: int(NAME_ANCHOR_MAX + 1) }); + if (rows.length > NAME_ANCHOR_MAX) return { ...all, filter: 'v.name =~ $nameRe', params: { nameRe: wildcardRegex(text) } }; + names = rows.map((r) => r.name); + } else { + const typed = [...new Set([text, text.toLowerCase()])]; + names = (await read(`MATCH (c:Crate) WHERE c.name IN $names RETURN c.name AS name`, { names: typed })).map((r) => r.name); + if (!names.length) { + names = (await read(`MATCH (c:Crate) WHERE toLower(c.name) = $q RETURN c.name AS name LIMIT 1`, { q: text.toLowerCase() })).map((r) => r.name); + } + } + return { match: 'MATCH (nc:Crate WHERE nc.name IN $crateNames)-[:VERSION_OF]->(v:Version)', filter: 'true', params: { crateNames: names }, crates: names.length }; +}; diff --git a/render/server/metadata.mjs b/render/server/metadata.mjs index 5f69868..b0e2824 100644 --- a/render/server/metadata.mjs +++ b/render/server/metadata.mjs @@ -1,6 +1,7 @@ // `scope` is 'latest' (idx_is_latest) or 'all' (a label scan, seconds on millions of versions). A generate-metadata // run may be in progress, so any property can be null; the meta_* markers are what "scanned" means. import neo4j from 'neo4j-driver'; +import { crateNameScope, hasWildcard, wildcardRegex } from './match.mjs'; const scopeClause = (scope) => (scope === 'all' ? 'true' : 'v.latest = true'); const tri = (expr, value) => `($${value} = 'any' OR ($${value} = 'yes' AND ${expr}) OR ($${value} = 'no' AND NOT ${expr}))`; @@ -66,20 +67,23 @@ export const buildLinks = async (read, scope, limit) => ); export const buildTop = async (read, { scope, sort, desc, q, links, link, custom, minLines, limit, offset }) => { + const ns = await crateNameScope(read, neo4j.int, q); const order = BUILD_SORTS[sort] ?? BUILD_SORTS.lines; - // An exact `link` name is served by idx_version_links; otherwise idx_has_build_script. + // An exact `link` name is served by idx_version_links; a `*` / `?` pattern and no link use idx_has_build_script. + const linkPattern = link && hasWildcard(link); + const linkClause = !link ? '' : linkPattern ? 'v.links =~ $linkRe AND ' : 'v.links = $link AND '; const rows = await read( - `MATCH (v:Version) - WHERE ${link ? 'v.links = $link AND v.has_build_script = true' : 'v.has_build_script = true'} AND ${scopeClause(scope)} + `${ns.match} + WHERE ${linkClause}v.has_build_script = true AND ${scopeClause(scope)} AND v.meta_build IS NOT NULL - AND ($q = '' OR toLower(v.name) CONTAINS $q) + AND ${ns.filter} AND ${tri('v.links IS NOT NULL', 'links')} AND ${tri("v.build_script <> 'build.rs'", 'custom')} AND coalesce(v.build_script_lines, 0) >= $minLines WITH v ORDER BY ${order} ${desc ? 'DESC' : 'ASC'}, v.name, v.version SKIP $offset LIMIT $limit ${BUILD_ROW}`, - { q: q.toLowerCase(), links, link, custom, minLines: neo4j.int(minLines), offset: neo4j.int(offset), limit: neo4j.int(limit + 1) }, + { ...ns.params, links, link, linkRe: linkPattern ? wildcardRegex(link) : null, custom, minLines: neo4j.int(minLines), offset: neo4j.int(offset), limit: neo4j.int(limit + 1) }, ); return page(rows, limit); }; @@ -148,11 +152,12 @@ export const nativeStats = async (read, scope) => { }; export const nativeTop = async (read, { scope, sort, desc, q, vendored, prebuilt, lang, minLines, limit, offset }) => { + const ns = await crateNameScope(read, neo4j.int, q); const order = NATIVE_SORTS[sort] ?? NATIVE_SORTS.lines; const rows = await read( - `MATCH (v:Version) + `${ns.match} WHERE v.has_native_code = true AND ${scopeClause(scope)} - AND ($q = '' OR toLower(v.name) CONTAINS $q) + AND ${ns.filter} AND ${tri('v.vendored_native = true', 'vendored')} AND ${tri('v.native_prebuilt_files > 0', 'prebuilt')} AND ($lang = '' OR $lang IN coalesce(v.native_langs, [])) @@ -160,7 +165,7 @@ export const nativeTop = async (read, { scope, sort, desc, q, vendored, prebuilt WITH v ORDER BY ${order} ${desc ? 'DESC' : 'ASC'}, v.name, v.version SKIP $offset LIMIT $limit ${NATIVE_ROW}`, - { q: q.toLowerCase(), vendored, prebuilt, lang, minLines: neo4j.int(minLines), offset: neo4j.int(offset), limit: neo4j.int(limit + 1) }, + { ...ns.params, vendored, prebuilt, lang, minLines: neo4j.int(minLines), offset: neo4j.int(offset), limit: neo4j.int(limit + 1) }, ); return page(rows, limit); }; @@ -220,14 +225,15 @@ export const diffStats = async (read, scope) => { // Sorting by `added` is index-backed (idx_diff_lines_added); the computed sorts walk every scanned version in scope. export const diffTop = async (read, { scope, sort, desc, q, firsts, approximate, minChurn, limit, offset }) => { + const ns = await crateNameScope(read, neo4j.int, q); const order = DIFF_SORTS[sort] ?? DIFF_SORTS.churn; const firstsClause = firsts === 'include' ? 'true' : firsts === 'only' ? 'v.diff_base IS NULL' : 'v.diff_base IS NOT NULL'; const rows = await read( - `MATCH (v:Version) + `${ns.match} WHERE ${scopeClause(scope)} AND v.meta_diff IS NOT NULL AND ${firstsClause} - AND ($q = '' OR toLower(v.name) CONTAINS $q) + AND ${ns.filter} AND ${tri('v.diff_approximate = true', 'approximate')} ${DIFF_DERIVED} WHERE churn >= $minChurn @@ -235,7 +241,7 @@ export const diffTop = async (read, { scope, sort, desc, q, firsts, approximate, ORDER BY ${order} ${desc ? 'DESC' : 'ASC'}, v.name, v.version SKIP $offset LIMIT $limit ${DIFF_ROW}`, - { q: q.toLowerCase(), approximate, minChurn: neo4j.int(minChurn), offset: neo4j.int(offset), limit: neo4j.int(limit + 1) }, + { ...ns.params, approximate, minChurn: neo4j.int(minChurn), offset: neo4j.int(offset), limit: neo4j.int(limit + 1) }, ); return page(rows, limit); }; diff --git a/render/server/overview.mjs b/render/server/overview.mjs new file mode 100644 index 0000000..616a653 --- /dev/null +++ b/render/server/overview.mjs @@ -0,0 +1,289 @@ +// Default view payload: one crate, the dependencies its subject version declares, its newest versions, the +// crates its compiled code calls into, and (on request) its most-downloaded dependents. Everything comes +// back in one Cypher round trip (the port-forwarded database costs ~150 ms per statement), every part is +// bounded, and nodes are annotated with `role` / `counts` so the client can lay the graph out without +// a force simulation. +// +// Dependencies are folded to one edge per (from, crate): a version can declare the same crate several +// times (per target, per kind), which drew doubled arrows. The edge keeps every requirement, the strongest +// kind (normal > build > dev), `optional` only when every declaration is optional, and `via` (the version +// that declared it). Calls into a declared dependency land on that edge as `calls`; calls into other crates +// (inlined transitive code) become an `INVOKES` edge to a `callee` node. Self-calls are dropped. +// +// Dependents come from the Map precompute (`LOD_DEP`, one edge per crate pair, `versions` = how many +// versions declare it) ordered by downloads, so "the 25 biggest users of serde" is a 130k-edge scan +// (~350 ms) instead of a 1.7M-edge DEPENDS_ON walk. Inbound call counts are not offered here: even +// bounded, "who calls into serde" is 4-15 s (see the Callers view). + +export const OVERVIEW_PARTS = ['deps', 'versions', 'calls', 'dependents']; +export const OVERVIEW_LIMITS = { deps: 200, versions: 60, calls: 100, dependents: 100 }; +export const OVERVIEW_DEFAULTS = { deps: 60, versions: 12, calls: 40, dependents: 25 }; + +const OVERVIEW_CYPHER = ` +MATCH (c:Crate {name: $name}) +CALL { + WITH c + OPTIONAL MATCH (c)-[:VERSION_OF]->(v:Version) + WHERE $version IS NULL OR v.version = $version + WITH v + ORDER BY coalesce(v.latest, false) DESC, + v.semver_major DESC, v.semver_minor DESC, v.semver_patch DESC, + coalesce(v.semver_pre, '') = '' DESC, v.version DESC + LIMIT 1 + RETURN v AS subject +} +CALL { + WITH c + OPTIONAL MATCH (c)-[vo:VERSION_OF]->(v:Version) WHERE $wantVersions + WITH vo, v + ORDER BY coalesce(v.latest, false) DESC, + v.semver_major DESC, v.semver_minor DESC, v.semver_patch DESC, + coalesce(v.semver_pre, '') = '' DESC, v.version DESC + SKIP $skip LIMIT $versionsLimit + RETURN collect({vo: vo, v: v}) AS versions +} +CALL { + WITH subject + OPTIONAL MATCH (subject)-[d:DEPENDS_ON]->(dc:Crate) WHERE $wantDeps + WITH dc, collect(properties(d)) AS ds + WHERE dc IS NOT NULL + WITH dc, ds ORDER BY coalesce(dc.downloads, 0) DESC + WITH collect({dc: dc, ds: ds, + dependents: COUNT { (dc)<-[:LOD_DEP]-() }, + deps: COUNT { (dc)-[:LOD_DEP]->() }, + versions: COUNT { (dc)-[:VERSION_OF]->() }}) AS items + RETURN items[0..$depsLimit] AS deps, size(items) AS depsTotal +} +CALL { + WITH subject + OPTIONAL MATCH (subject)-[:HAS_FN]->(f:Fn)-[k:CALLS]->(s:Symbol)-[:IN]->(ic:Crate) WHERE $wantCalls + WITH ic, count(k) AS calls, count(DISTINCT f) AS callers, collect(DISTINCT s.name) AS targets + WHERE ic IS NOT NULL + WITH ic, calls, callers, size(targets) AS symbols, targets[0..10] AS targets + ORDER BY calls DESC + WITH collect({ic: ic, calls: calls, callers: callers, symbols: symbols, targets: targets}) AS items + RETURN items[0..$callsLimit] AS calls, size(items) AS callsTotal +} +CALL { + WITH c + OPTIONAL MATCH (c)<-[l:LOD_DEP]-(d:Crate) WHERE $wantDependents + WITH d, l WHERE d IS NOT NULL + WITH d, l ORDER BY coalesce(d.downloads, 0) DESC + SKIP $skip LIMIT $dependentsLimit + RETURN collect({d: d, versions: l.versions, + dependents: COUNT { (d)<-[:LOD_DEP]-() }, + deps: COUNT { (d)-[:LOD_DEP]->() }, + versionsCount: COUNT { (d)-[:VERSION_OF]->() }}) AS dependents +} +RETURN c, subject, + {versions: COUNT { (c)-[:VERSION_OF]->() }, + dependents: COUNT { (c)<-[:LOD_DEP]-() }, + dependentVersions: COUNT { (c)<-[:DEPENDS_ON]-() }, + deps: COUNT { (c)-[:LOD_DEP]->() }, + symbolsCalled: COUNT { (c)<-[:IN]-() }} AS counts, + versions, deps, depsTotal, calls, callsTotal, dependents`; + +// Without a Map build there is no LOD_DEP; sample distinct dependents off DEPENDS_ON instead (stops at the +// limit, so it is bounded even for serde, but it is not ordered by downloads). +const FALLBACK_DEPENDENTS_CYPHER = ` +MATCH (c:Crate {name: $name})<-[:DEPENDS_ON]-(:Version)<-[:VERSION_OF]-(d:Crate) +WITH DISTINCT d LIMIT $dependentsLimit +RETURN collect({d: d, versions: null, dependents: 0, deps: 0, + versionsCount: COUNT { (d)-[:VERSION_OF]->() }}) AS dependents`; + +const KIND_RANK = { normal: 0, build: 1, dev: 2 }; +const normKind = (k) => String(k ?? 'normal').toLowerCase(); + +const uniq = (xs) => [...new Set(xs.filter((x) => x !== null && x !== undefined && x !== ''))]; + +// One DEPENDS_ON per crate: the strongest kind wins, optional only if every declaration is optional. +export const mergeDeclarations = (ds) => { + let kind = 'dev'; + let optional = true; + for (const d of ds) { + const k = normKind(d.kind); + if ((KIND_RANK[k] ?? 0) < (KIND_RANK[kind] ?? 0)) kind = k; + if (d.optional !== true) optional = false; + } + if (ds.length === 0) { + kind = 'normal'; + optional = false; + } + const requirements = uniq(ds.map((d) => d.requirement)); + const features = uniq(ds.flatMap((d) => (Array.isArray(d.features) ? d.features : []))); + return { + kind, + optional, + requirement: requirements.join(' | '), + features, + declarations: ds.length, + }; +}; + +const parseParts = (value) => { + const raw = Array.isArray(value) ? value : String(value ?? '').split(','); + const parts = raw.map((p) => String(p).trim()).filter((p) => OVERVIEW_PARTS.includes(p)); + return parts.length ? [...new Set(parts)] : ['deps', 'versions', 'calls']; +}; + +const clampTo = (value, fallback, max) => { + const n = Number.parseInt(value ?? '', 10); + if (!Number.isFinite(n) || n <= 0) return fallback; + return Math.min(n, max); +}; + +/** + * Builds the overview payload. `readGraph(cypher, params)` is the server's read helper; `int` wraps numbers + * for the driver. Options: `name` (exact crate name), `parts`, `limit` (per part), `skip` (versions / + * dependents paging), `version` (subject version; default latest), `from` ('crate' | 'version': which node + * the dependency / call edges start at), `root` (mark the crate as the layout root). + */ +export async function crateOverview(readGraph, int, opts) { + const parts = parseParts(opts.parts); + const want = (p) => parts.includes(p); + const limits = { + deps: clampTo(opts.limit, OVERVIEW_DEFAULTS.deps, OVERVIEW_LIMITS.deps), + versions: clampTo(opts.limit, OVERVIEW_DEFAULTS.versions, OVERVIEW_LIMITS.versions), + calls: clampTo(opts.limit, OVERVIEW_DEFAULTS.calls, OVERVIEW_LIMITS.calls), + dependents: clampTo(opts.limit, OVERVIEW_DEFAULTS.dependents, OVERVIEW_LIMITS.dependents), + }; + const skip = Math.max(0, Number.parseInt(opts.skip ?? '0', 10) || 0); + const version = typeof opts.version === 'string' && opts.version ? opts.version : null; + const fromVersion = opts.from === 'version' && version !== null; + + const started = Date.now(); + const g = await readGraph(OVERVIEW_CYPHER, { + name: opts.name, + version, + skip: int(skip), + wantVersions: want('versions'), + wantDeps: want('deps'), + wantCalls: want('calls'), + wantDependents: want('dependents'), + versionsLimit: int(limits.versions), + depsLimit: int(limits.deps), + callsLimit: int(limits.calls), + dependentsLimit: int(limits.dependents), + }); + const row = g.rows[0]; + if (!row || !row.c) return null; + + const byId = new Map(g.nodes.map((n) => [n.id, n])); + const crate = byId.get(row.c.id); + const subject = row.subject ? byId.get(row.subject.id) : null; + const subjectVersion = subject ? String(subject.properties.version ?? '') : null; + const fromId = fromVersion && subject ? subject.id : crate.id; + + const nodes = new Map(); + const rels = []; + const keep = (node, role, counts) => { + if (!node) return; + const existing = nodes.get(node.id); + if (existing) { + if (counts && !existing.counts) existing.counts = counts; + return; + } + const copy = { ...node }; + if (role) copy.role = role; + if (counts) copy.counts = counts; + nodes.set(node.id, copy); + }; + + keep(crate, opts.root ? 'root' : undefined, row.counts); + if (fromVersion && subject) keep(subject); + + for (const item of row.versions ?? []) { + if (!item.v) continue; + keep(byId.get(item.v.id)); + if (item.vo) rels.push(item.vo); + } + + // Calls keyed by callee crate id; the ones into declared dependencies decorate the dependency edge. + const calls = new Map(); + let selfCalls = 0; + for (const item of row.calls ?? []) { + if (!item.ic) continue; + if (item.ic.id === crate.id) { + selfCalls += item.calls; + continue; + } + calls.set(item.ic.id, item); + } + + const depIds = new Set(); + for (const item of row.deps ?? []) { + if (!item.dc || item.dc.id === crate.id) continue; + depIds.add(item.dc.id); + keep(byId.get(item.dc.id), 'dep', { dependents: item.dependents, deps: item.deps, versions: item.versions }); + const merged = mergeDeclarations(item.ds ?? []); + const call = calls.get(item.dc.id); + rels.push({ + id: `dep:${fromId}->${item.dc.id}`, + type: 'DEPENDS_ON', + from: fromId, + to: item.dc.id, + properties: { + ...merged, + via: subjectVersion, + ...(call ? { calls: call.calls, callers: call.callers, symbols: call.symbols, targets: call.targets } : {}), + }, + }); + } + + for (const [id, call] of calls) { + if (depIds.has(id)) continue; + keep(byId.get(id), 'callee'); + rels.push({ + id: `agg:${fromId}->${id}:INVOKES`, + type: 'INVOKES', + from: fromId, + to: id, + properties: { count: call.calls, callers: call.callers, symbols: call.symbols, targets: call.targets, aggregated: true, via: subjectVersion }, + }); + } + + let dependents = row.dependents ?? []; + let dependentsSource = 'lod'; + if (want('dependents') && dependents.filter((d) => d.d).length === 0 && row.counts.dependents === 0 && row.counts.dependentVersions > 0) { + const fb = await readGraph(FALLBACK_DEPENDENTS_CYPHER, { name: opts.name, dependentsLimit: int(limits.dependents) }); + for (const n of fb.nodes) byId.set(n.id, n); + dependents = fb.rows[0]?.dependents ?? []; + dependentsSource = 'sample'; + } + for (const item of dependents) { + if (!item.d || item.d.id === crate.id) continue; + keep(byId.get(item.d.id), 'dependent', { dependents: item.dependents, deps: item.deps, versions: item.versionsCount }); + rels.push({ + id: `dep:${item.d.id}->${crate.id}`, + type: 'DEPENDS_ON', + from: item.d.id, + to: crate.id, + properties: { aggregated: true, versions: item.versions ?? null }, + }); + } + + const nodeIds = new Set(nodes.keys()); + return { + nodes: [...nodes.values()], + rels: rels.filter((r) => nodeIds.has(r.from) && nodeIds.has(r.to)), + meta: { + records: g.meta.records, + elapsedMs: Date.now() - started, + resolvedName: String(crate.properties.name), + overview: { + name: String(crate.properties.name), + version: subjectVersion, + parts, + from: fromVersion ? 'version' : 'crate', + skip, + limits, + counts: row.counts, + depsTotal: row.depsTotal ?? 0, + callsTotal: row.callsTotal ?? 0, + selfCalls, + dependentsShown: dependents.filter((d) => d.d).length, + dependentsSource: want('dependents') ? dependentsSource : null, + }, + }, + }; +} diff --git a/render/server/registry.mjs b/render/server/registry.mjs index 1d0f95a..5adf086 100644 --- a/render/server/registry.mjs +++ b/render/server/registry.mjs @@ -1,6 +1,7 @@ // Dates are ISO strings (`toString(datetime)`), never driver temporal objects. Queries are index-backed // and LIMITed; only the publisher leaderboards aggregate a whole relationship type, so the routes cache them. import neo4j from 'neo4j-driver'; +import { hasWildcard, nameFilter, nameParams, wildcardRegex } from './match.mjs'; const page = (rows, limit) => { const hasMore = rows.length > limit; @@ -63,7 +64,23 @@ export const registryStats = async (read) => { }; // `STARTS WITH` is case-sensitive and served by idx_user_login, so try the typed, lowercase and capitalised spellings. +// A `*` / `?` query is a pattern over every login (73k users, 1.5k teams: a cheap scan), ranked by activity. export const searchUsers = async (read, q, limit) => { + if (hasWildcard(q)) { + const re = wildcardRegex(q); + const [users, teams] = await Promise.all( + ['User', 'Team'].map((label) => + read( + `MATCH (u:${label}) WHERE u.login =~ $re + WITH u, COUNT { (u)-[:PUBLISHED]->(:Crate) } AS crates, COUNT { (u)-[:OWNS]->() } AS owns + RETURN u.login AS login, u.name AS name, crates, owns + ORDER BY crates + owns DESC, u.login LIMIT $limit`, + { re, limit: neo4j.int(limit) }, + ), + ), + ); + return { users, teams }; + } const variants = [...new Set([q, q.toLowerCase(), q.toUpperCase(), q.charAt(0).toUpperCase() + q.slice(1)])]; const [users, teams] = await Promise.all([ read( @@ -225,8 +242,8 @@ const VERSION_RELEASE_ROW = ` // `extra.joined` is true when the crate already existed (a new publisher). export const releases = async (read, { kind, source, from, to, q, minDownloads, sort, desc, limit, offset }) => { const dir = desc ? 'DESC' : 'ASC'; - const params = { from, to, q: q.toLowerCase(), minDownloads: neo4j.int(minDownloads), offset: neo4j.int(offset), limit: neo4j.int(limit + 1) }; - const filters = (nameExpr, dlExpr) => `AND ($q = '' OR toLower(${nameExpr}) CONTAINS $q) AND coalesce(${dlExpr}, 0) >= $minDownloads`; + const params = { from, to, ...nameParams(q), minDownloads: neo4j.int(minDownloads), offset: neo4j.int(offset), limit: neo4j.int(limit + 1) }; + const filters = (nameExpr, dlExpr) => `AND ${nameFilter(nameExpr)} AND coalesce(${dlExpr}, 0) >= $minDownloads`; let cypher; if (kind === 'crates' && source === 'created') { const order = { date: 'c.created_at', downloads: 'c.downloads', name: 'c.name' }[sort]; diff --git a/render/server/unsafe.mjs b/render/server/unsafe.mjs index 43bf260..c74fd10 100644 --- a/render/server/unsafe.mjs +++ b/render/server/unsafe.mjs @@ -1,4 +1,5 @@ import neo4j from 'neo4j-driver'; +import { crateNameScope } from './match.mjs'; // `scope` 'latest' is served by idx_is_latest; 'all' is a label scan (seconds on millions of versions). @@ -65,12 +66,13 @@ export const unsafeStats = async (read, scope) => { // Returns one row more than `limit` when more exist (`hasMore`). export const unsafeTop = async (read, { scope, sort, desc, q, forbids, build, minLines, minUnsafe, limit, offset }) => { + const ns = await crateNameScope(read, neo4j.int, q); const order = SORTS[sort] ?? SORTS.total; const rows = await read( - `MATCH (v:Version) + `${ns.match} WHERE ${scopeClause(scope)} AND v.unsafe_scanner IS NOT NULL AND v.unsafe_error IS NULL - AND ($q = '' OR toLower(v.name) CONTAINS $q) + AND ${ns.filter} AND ($forbids = 'any' OR ($forbids = 'yes' AND v.forbids_unsafe = true) OR ($forbids = 'no' AND coalesce(v.forbids_unsafe, false) = false)) AND ($build = 'any' OR v.has_build_script = ($build = 'yes')) @@ -80,7 +82,7 @@ export const unsafeTop = async (read, { scope, sort, desc, q, forbids, build, mi ORDER BY ${order} ${desc ? 'DESC' : 'ASC'}, v.name, v.version SKIP $offset LIMIT $limit ${RETURN_ROW}`, - { q: q.toLowerCase(), forbids, build, minLines: neo4j.int(minLines), minUnsafe: neo4j.int(minUnsafe), offset: neo4j.int(offset), limit: neo4j.int(limit + 1) }, + { ...ns.params, forbids, build, minLines: neo4j.int(minLines), minUnsafe: neo4j.int(minUnsafe), offset: neo4j.int(offset), limit: neo4j.int(limit + 1) }, ); const hasMore = rows.length > limit; return { rows: hasMore ? rows.slice(0, limit) : rows, hasMore }; diff --git a/render/src/App.tsx b/render/src/App.tsx index 9bc4e22..24d3408 100644 --- a/render/src/App.tsx +++ b/render/src/App.tsx @@ -8,19 +8,24 @@ import { type CrateLoadOptions, type ExpandType, type GraphNode, - type GraphPayload, type GraphRel, type Health, + type OverviewMeta, + type OverviewPart, type Stats, type View, } from './api'; -import { emptyStore, mergePayload, removeNode, toNvl, type GraphStore } from './graph'; -import { useTouchZoomPan } from './touch'; +import { emptyStore, isCrate, isVersion, mergePayload, removeNode, toNvl, type GraphStore } from './graph'; +import { DEFAULT_DEPS_OPTIONS, type DepsOptions, type DepsStatus } from './deps'; +import { clearOverviewLayout, mergeOverview, overviewLabels, placeOverview, shownCount, toNvlOverview } from './overview'; +import { OverviewLabels } from './components/OverviewLabels'; +import { useIsMobile, useTouchZoomPan } from './touch'; import { Toolbar } from './components/Toolbar'; import { DetailsPanel } from './components/DetailsPanel'; import { CypherPanel } from './components/CypherPanel'; import { StatusBar } from './components/StatusBar'; import { MapView } from './components/MapView'; +import { DepsView } from './components/DepsView'; import { CallersView } from './components/CallersView'; import { UnsafeView } from './components/UnsafeView'; import { BuildView } from './components/BuildView'; @@ -31,7 +36,6 @@ import { ReleasesView } from './components/ReleasesView'; import { SupplyView } from './components/SupplyView'; const DEFAULT_OPTIONS: CrateLoadOptions = { versions: 'latest', deps: true, invokes: true, dependents: false, limit: 200 }; -const DEFAULT_DEPS_LIMIT = 5000; const errorMessage = (err: unknown): string => { if (err instanceof ApiError) return err.code ? `${err.message} [${err.code}]` : err.message; @@ -43,10 +47,14 @@ export function App() { const nvlRef = useRef(null); const [canvasEl, setCanvasEl] = useState(null); useTouchZoomPan(canvasEl, nvlRef); + // On a phone the details sheet covers the canvas, so a fresh load leaves nothing selected there. + const isMobile = useIsMobile(); const [store, setStore] = useState(emptyStore); const [selectedId, setSelectedId] = useState(null); const [options, setOptions] = useState(DEFAULT_OPTIONS); - const [layout, setLayout] = useState('forceDirected'); + // The Default view lays nodes out itself (src/overview.ts) on NVL's free layout; other graph views simulate. + const [layout, setLayout] = useState('free'); + const [overviewMeta, setOverviewMeta] = useState<(OverviewMeta & { elapsedMs: number }) | null>(null); const [busy, setBusy] = useState(false); const [error, setError] = useState(null); const [health, setHealth] = useState(null); @@ -55,9 +63,9 @@ export function App() { const [hiddenRelTypes, setHiddenRelTypes] = useState>(new Set()); const [showRelCaptions, setShowRelCaptions] = useState(false); const [view, setView] = useState('default'); - const [depsLimit, setDepsLimit] = useState(DEFAULT_DEPS_LIMIT); - const [depsDepth, setDepsDepth] = useState(0); - const [depsMeta, setDepsMeta] = useState(null); + const [depsOptions, setDepsOptions] = useState(DEFAULT_DEPS_OPTIONS); + const [depsTarget, setDepsTarget] = useState<{ name: string; nonce: number } | null>(null); + const [depsStatus, setDepsStatus] = useState(null); const [mapTarget, setMapTarget] = useState<{ name: string; nonce: number } | null>(null); const [mapBudget, setMapBudget] = useState(600); const [mapMeta, setMapMeta] = useState<{ mode: string; level: number; inView: number; total: number; nodes: number; rels: number; elapsedMs: number } | null>(null); @@ -101,77 +109,62 @@ export function App() { // Resolved name: the server accepts partial input ("serd" loads "serde"). const [loadedName, setLoadedName] = useState(null); + // Replaces the canvas: the overview is laid out around one root (src/overview.ts); expansions merge into it. const loadCrate = useCallback( async (input: string) => { setBusy(true); setError(null); try { - const payload = await api.loadCrate(input, options); + const parts: OverviewPart[] = ['deps', 'versions']; + if (options.invokes) parts.push('calls'); + if (options.dependents) parts.push('dependents'); + const payload = await api.overview({ name: input, parts, root: true }); const name = payload.meta.resolvedName ?? input; setLoadedName(name); - setStore((s) => mergePayload(s, payload, name)); + setOverviewMeta(payload.meta.overview ? { ...payload.meta.overview, elapsedMs: payload.meta.elapsedMs } : null); const crate = payload.nodes.find((n) => n.labels.includes('Crate') && n.properties.name === name); - if (crate) setSelectedId(crate.id); - window.setTimeout(fitAll, 400); + clearOverviewLayout(); + const next = mergePayload(emptyStore(), payload, name); + placeOverview(next, crate?.id ?? null); + setStore(next); + setSelectedId(isMobile ? null : (crate?.id ?? null)); + window.setTimeout(fitAll, 150); } catch (err) { setError(errorMessage(err)); } finally { setBusy(false); } }, - [options, fitAll], + [options, fitAll, isMobile], ); - const loadDeps = useCallback( - async (crate: string | null, direction: 'in' | 'out' | 'both' = 'both', replace = false) => { + // One bounded part (deps / versions / calls / dependents) of one crate, hung off its node (or one of its + // version nodes when `version` is given) and placed away from the root. + const expandOverview = useCallback( + async (name: string, part: OverviewPart, opts: { version?: string } = {}) => { setBusy(true); setError(null); try { - const payload = await api.loadDeps(crate, direction, depsLimit); - const name = payload.meta.resolvedName ?? crate ?? null; - if (name) setLoadedName(name); - setDepsMeta(payload.meta); - setStore((s) => mergePayload(replace ? emptyStore() : s, payload, name ?? undefined)); - if (name) { - const node = payload.nodes.find((n) => n.labels.includes('Crate') && n.properties.name === name); - if (node) setSelectedId(node.id); - } else { - setSelectedId(null); + const skip = part === 'versions' || part === 'dependents' ? shownCount(store, name, part) : 0; + const payload = await api.overview({ name, parts: [part], version: opts.version ?? null, from: opts.version ? 'version' : 'crate', skip }); + const anchor = opts.version + ? payload.nodes.find((n) => isVersion(n) && n.properties.name === name && n.properties.version === opts.version) + : payload.nodes.find((n) => isCrate(n) && n.properties.name === name); + setStore((s) => { + const next = mergeOverview(s, payload); + placeOverview(next, anchor?.id ?? null); + return next; + }); + if (payload.nodes.length <= 1 && payload.rels.length === 0) { + setError(`${name}${opts.version ? `@${opts.version}` : ''}: nothing more to add for ${part}`); } - window.setTimeout(fitAll, 600); - } catch (err) { - setError(errorMessage(err)); - } finally { - setBusy(false); - } - }, - [depsLimit, fitAll], - ); - - // Replaces the canvas; expansions from the details panel merge via loadDeps. - const loadChain = useCallback( - async (q: string, exact = false) => { - setBusy(true); - setError(null); - try { - const payload = await api.loadDepsChain(q, depsLimit, depsDepth, exact); - setDepsMeta(payload.meta); - let next = emptyStore(); - for (const r of payload.meta.roots ?? []) next.roots.add(r); - next = mergePayload(next, payload); - setStore(next); - setLoadedName(null); - const roots = payload.meta.roots ?? []; - const single = roots.length === 1 ? payload.nodes.find((n) => n.properties.name === roots[0]) : undefined; - setSelectedId(single?.id ?? null); - window.setTimeout(fitAll, 600); } catch (err) { setError(errorMessage(err)); } finally { setBusy(false); } }, - [depsLimit, depsDepth, fitAll], + [store], ); const switchView = useCallback( @@ -181,34 +174,36 @@ export function App() { setStore(emptyStore()); setSelectedId(null); setError(null); - setDepsMeta(null); + setDepsStatus(null); + setOverviewMeta(null); + clearOverviewLayout(); + setLayout((l) => (next === 'default' ? 'free' : l === 'free' ? 'forceDirected' : l)); }, [view], ); const load = useCallback( - (input: string, exact = false) => { + (input: string) => { if (view === 'map') { setMapTarget({ name: input, nonce: Date.now() }); return Promise.resolve(); } - return view === 'deps' ? loadChain(input, exact) : loadCrate(input); + if (view === 'deps') { + setDepsTarget({ name: input, nonce: Date.now() }); + return Promise.resolve(); + } + return loadCrate(input); }, - [view, loadChain, loadCrate], + [view, loadCrate], ); - const openInDeps = useCallback( - (name: string) => { - setView('deps'); - setStore(emptyStore()); - setSelectedId(null); - setError(null); - setDepsMeta(null); - // loadChain reads view-independent state, so it can run right after the switch. - void loadChain(name, true); - }, - [loadChain], - ); + const openInDeps = useCallback((name: string) => { + setView('deps'); + setStore(emptyStore()); + setSelectedId(null); + setError(null); + setDepsTarget({ name, nonce: Date.now() }); + }, []); const openInGraph = useCallback( (name: string) => { @@ -216,6 +211,7 @@ export function App() { setStore(emptyStore()); setSelectedId(null); setError(null); + setLayout('free'); void loadCrate(name); }, [loadCrate], @@ -232,20 +228,30 @@ export function App() { setError(null); try { const payload = await api.expand(id, direction, types, 100); - setStore((s) => mergePayload(s, payload)); + setStore((s) => { + const next = mergePayload(s, payload); + // The Default view's free layout needs coordinates for owners / uploaders too. + if (view === 'default') placeOverview(next, id); + return next; + }); } catch (err) { setError(errorMessage(err)); } finally { setBusy(false); } - }, []); + }, [view]); const runCypher = useCallback(async (query: string) => { setBusy(true); setError(null); try { const payload = await api.cypher(query); - setStore((s) => mergePayload(s, payload)); + setStore((s) => { + const next = mergePayload(s, payload); + // The free layout needs coordinates; ring the newcomers around whatever is drawn. + if (view === 'default') placeOverview(next, null); + return next; + }); window.setTimeout(fitAll, 400); return payload; } catch (err) { @@ -254,7 +260,7 @@ export function App() { } finally { setBusy(false); } - }, [fitAll]); + }, [fitAll, view]); const remove = useCallback((id: string) => { setStore((s) => removeNode(s, id)); @@ -265,12 +271,18 @@ export function App() { setStore(emptyStore()); setSelectedId(null); setError(null); + setOverviewMeta(null); + clearOverviewLayout(); }, []); const { nodes, rels } = useMemo( - () => toNvl(store, selectedId, { showRelCaptions, hiddenRelTypes }), - [store, selectedId, showRelCaptions, hiddenRelTypes], + () => + view === 'default' + ? toNvlOverview(store, selectedId, { showRelCaptions, hiddenRelTypes }) + : toNvl(store, selectedId, { showRelCaptions, hiddenRelTypes }), + [store, selectedId, showRelCaptions, hiddenRelTypes, view], ); + const labelSpecs = useMemo(() => (view === 'default' ? overviewLabels(store, overviewMeta?.version ?? null) : []), [store, view, overviewMeta]); const selected: GraphNode | GraphRel | null = selectedId ? (store.nodes.get(selectedId) ?? store.rels.get(selectedId) ?? null) : null; @@ -304,9 +316,11 @@ export function App() { onRelationshipClick: (rel: NvlRel) => setSelectedId(rel.id), onCanvasClick: () => setSelectedId(null), onNodeDoubleClick: (node: NvlNode) => { - if (view === 'deps') { - const n = store.nodes.get(node.id); - if (n) void loadDeps(String(n.properties.name), 'both'); + const n = store.nodes.get(node.id); + if (view === 'default' && n && isCrate(n)) { + void expandOverview(String(n.properties.name), 'deps'); + } else if (view === 'default' && n && isVersion(n)) { + void expandOverview(String(n.properties.name), 'deps', { version: String(n.properties.version) }); } else { void expand(node.id, 'both', []); } @@ -317,7 +331,7 @@ export function App() { onZoomAndPan: true, onDrag: true, }), - [expand, view, store, loadDeps], + [expand, expandOverview, view, store], ); return ( @@ -325,16 +339,13 @@ export function App() { void load(n, exact)} + onLoad={(n) => void load(n)} loadedName={loadedName} onClear={clear} onFit={fitAll} @@ -344,7 +355,7 @@ export function App() { />
{view === 'callers' ? ( - + ) : view === 'unsafe' ? ( ) : view === 'build' ? ( @@ -368,32 +379,32 @@ export function App() { onOpenInDeps={openInDeps} onMeta={setMapMeta} /> + ) : view === 'deps' ? ( + ) : ( <>
- {nodes.length === 0 && view === 'deps' ? ( + {nodes.length === 0 ? (
-

crate dependency chain

+

crate overview

{busy - ? 'Resolving the transitive dependency chain…' - : 'Type a crate name and press Load to draw its full transitive dependency chain. If no crate has exactly that name, 3 or more characters make every crate whose name contains the text a root.'} -

-

- Versions are folded into their crate: an edge A → B means some version of A depends on B. Max edges and max depth - (0 = unlimited) bound the chain. -

-
- ) : nodes.length === 0 ? ( -
-

painter graph

-

- Type part of a crate name above and press Load. Partial names match anywhere in the name and load the - closest crate, with its versions and their dependency / invocation edges. + ? 'Loading…' + : 'Type a crate name and press Load. You get the crate in the middle, what its latest version depends on to the right (normal deps inside, build / dev outside, dimmer when optional, thicker when its code calls into them), its newest versions along the bottom, and, with the dependents option, the biggest crates that use it on the left.'}

- Or open the Cypher panel at the bottom and run any read query: every node and relationship in the result is - drawn. + Click anything for details and bounded next steps (dependencies, dependents, older versions, calls, owners); + double-click a crate or version to add its dependencies. The Cypher panel below merges any read query into the + canvas.

) : null} @@ -407,6 +418,7 @@ export function App() { onInitializationError={(e) => setError(`NVL failed to initialise: ${errorMessage(e)}`)} style={{ width: '100%', height: '100%' }} /> +
diff --git a/render/src/components/NameFilter.tsx b/render/src/components/NameFilter.tsx new file mode 100644 index 0000000..739357b --- /dev/null +++ b/render/src/components/NameFilter.tsx @@ -0,0 +1,103 @@ +import { useEffect, useRef, useState } from 'react'; +import { api } from '../api'; +import { PATTERN_HINT } from '../match'; + +export interface Hint { + value: string; + note?: string; +} + +/** Crate-name hints: partial matches as you type (or the pattern's matches), most relevant first. */ +export const crateHints = (q: string): Promise => api.searchCrates(q, 12).then((names) => names.map((value) => ({ value }))); + +interface Props { + value: string; + onChange: (value: string) => void; + /** Hints for the typed text; always partial, so the user can pick an exact name. */ + hints?: (q: string) => Promise; + placeholder?: string; + className?: string; +} + +/** + * Text filter for the table views. The table matches the typed text exactly (case-insensitive) unless it + * contains `*` / `?`; the dropdown offers partial matches and picking one sets that exact name. + */ +export function NameFilter({ value, onChange, hints = crateHints, placeholder = `Crate: ${PATTERN_HINT}`, className = 'q' }: Props) { + const [items, setItems] = useState([]); + const [open, setOpen] = useState(false); + const [active, setActive] = useState(-1); + const timer = useRef(undefined); + + useEffect(() => { + window.clearTimeout(timer.current); + const q = value.trim(); + if (!open || !q) { + setItems([]); + return; + } + let cancelled = false; + timer.current = window.setTimeout(() => { + hints(q) + .then((r) => { + if (cancelled) return; + // An exact hit adds nothing to what is already in the box. + setItems(r.length === 1 && r[0].value.toLowerCase() === q.toLowerCase() ? [] : r); + setActive(-1); + }) + .catch(() => !cancelled && setItems([])); + }, 150); + return () => { + cancelled = true; + window.clearTimeout(timer.current); + }; + }, [value, open, hints]); + + const pick = (v: string) => { + setOpen(false); + onChange(v); + }; + + return ( +
+ { + onChange(e.target.value); + setOpen(true); + }} + onFocus={() => setOpen(true)} + onBlur={() => window.setTimeout(() => setOpen(false), 150)} + onKeyDown={(e) => { + if (e.key === 'ArrowDown') { + e.preventDefault(); + setActive((a) => Math.min(a + 1, items.length - 1)); + } else if (e.key === 'ArrowUp') { + e.preventDefault(); + setActive((a) => Math.max(a - 1, -1)); + } else if (e.key === 'Enter') { + if (active >= 0 && items[active]) pick(items[active].value); + else setOpen(false); + } else if (e.key === 'Escape') { + setOpen(false); + } + }} + spellCheck={false} + /> + {open && items.length > 0 && ( +
    + {items.map((h, i) => ( +
  • pick(h.value)}> + {h.value} + {h.note && {h.note}} +
  • + ))} +
+ )} +
+ ); +} diff --git a/render/src/components/NativeView.tsx b/render/src/components/NativeView.tsx index 66f7760..fce69ca 100644 --- a/render/src/components/NativeView.tsx +++ b/render/src/components/NativeView.tsx @@ -23,6 +23,7 @@ import { type Column, type HistoryBar, } from './meta'; +import { NameFilter } from './NameFilter'; interface Props { busy: boolean; @@ -220,7 +221,7 @@ export function NativeView({ busy, onBusy, onError, onOpenCrate }: Props) {
- setQ(e.target.value)} spellCheck={false} className="q" /> +