diff --git a/packages/@nitpicker/crawler/src/archive/database.spec.ts b/packages/@nitpicker/crawler/src/archive/database.spec.ts index 954ca6b0..450ffe83 100644 --- a/packages/@nitpicker/crawler/src/archive/database.spec.ts +++ b/packages/@nitpicker/crawler/src/archive/database.spec.ts @@ -3443,6 +3443,47 @@ describe('getJSON (getConfig 経由)', () => { await db2.destroy(); }); + + it('info.maxExcludedDepth が NULL の場合フォールバック値 0 を返す(issue #261)', async () => { + // `maxExcludedDepth` is a plain integer column, not a JSON column + // like `excludes`, so it needs its own NULL-fallback coverage — + // getJSON's fallback path above does not exercise it. + const nullDepthDbPath = path.resolve( + workingDir, + 'null-max-excluded-depth-test.sqlite', + ); + const db = await Database.connect({ filename: nullDepthDbPath }); + + const config: Config = { + version: '0.13.0', + name: 'test', + baseUrl: 'https://example.com', + roots: ['https://example.com'], + recursive: false, + interval: 500, + image: false, + fetchExternal: false, + parallels: 1, + excludes: [], + excludeKeywords: [], + excludeUrls: [], + maxExcludedDepth: 3, + retry: 3, + fromList: false, + disableQueries: false, + userAgent: 'test', + ignoreRobots: false, + }; + + await db.setConfig(config); + await db.getKnex()('info').update({ maxExcludedDepth: null }); + + const retrieved = await db.getConfig(); + expect(retrieved.maxExcludedDepth).toBe(0); + + await db.destroy(); + await remove(nullDepthDbPath); + }); }); describe('insertPageError', () => { diff --git a/packages/@nitpicker/crawler/src/archive/db-ops/config/get-config.ts b/packages/@nitpicker/crawler/src/archive/db-ops/config/get-config.ts index 73b32315..8f11a6c3 100644 --- a/packages/@nitpicker/crawler/src/archive/db-ops/config/get-config.ts +++ b/packages/@nitpicker/crawler/src/archive/db-ops/config/get-config.ts @@ -23,6 +23,7 @@ export async function getConfig(knex: Knex): Promise { excludeUrls: getJSON(config.excludeUrls, []), roots: getJSON(config.roots, []), retry: config.retry ?? 3, + maxExcludedDepth: config.maxExcludedDepth ?? 0, }; // @ts-expect-error — `id` is the primary key, not part of the public Config shape delete opt.id; diff --git a/packages/@nitpicker/query/src/get-summary-fast-path.spec.ts b/packages/@nitpicker/query/src/get-summary-fast-path.spec.ts index daa766ec..1ba2bd4f 100644 --- a/packages/@nitpicker/query/src/get-summary-fast-path.spec.ts +++ b/packages/@nitpicker/query/src/get-summary-fast-path.spec.ts @@ -25,6 +25,10 @@ function makeSummary(baseUrl: string): SummaryResult { return { baseUrl, roots: [], + excludes: [], + excludeKeywords: [], + excludeUrls: [], + maxExcludedDepth: 0, totalPages: 0, internalPages: 0, externalPages: 0, @@ -40,6 +44,8 @@ function makeSummary(baseUrl: string): SummaryResult { ogImage: 0, }, contentTypeDistribution: [], + networkOutageAffectedFailures: 0, + consoleLogCounts: { pageerror: 0, error: 0, warn: 0 }, }; } diff --git a/packages/@nitpicker/query/src/get-summary.spec.ts b/packages/@nitpicker/query/src/get-summary.spec.ts index ec96df5d..2b6548a0 100644 --- a/packages/@nitpicker/query/src/get-summary.spec.ts +++ b/packages/@nitpicker/query/src/get-summary.spec.ts @@ -1106,3 +1106,42 @@ describe('getSummary: console log counts (issue #228)', () => { rmSync(dir, { recursive: true, force: true }); }); }); + +describe('getSummary: exclude settings (issue #261)', () => { + it('passes through excludes/excludeKeywords/excludeUrls/maxExcludedDepth from config', async () => { + const dir = path.resolve(__dirname, '__test_fixtures_summary_excludes__'); + const archiveFilePath = path.resolve(dir, 'summary-excludes.nitpicker'); + const { mkdirSync, rmSync } = await import('node:fs'); + mkdirSync(dir, { recursive: true }); + const archive = await Archive.create({ filePath: archiveFilePath, cwd: dir }); + await archive.setConfig({ + baseUrl: 'https://example.com', + roots: ['https://example.com'], + name: 'test', + version: '0.13.0', + recursive: true, + interval: 0, + image: true, + fetchExternal: false, + parallels: 1, + excludes: ['/admin/*'], + excludeKeywords: ['draft'], + excludeUrls: ['https://example.com/temp'], + maxExcludedDepth: 3, + retry: 3, + fromList: false, + disableQueries: false, + userAgent: 'test', + ignoreRobots: false, + }); + + const result = await getSummary(archive); + expect(result.excludes).toEqual(['/admin/*']); + expect(result.excludeKeywords).toEqual(['draft']); + expect(result.excludeUrls).toEqual(['https://example.com/temp']); + expect(result.maxExcludedDepth).toBe(3); + + await archive.close(); + rmSync(dir, { recursive: true, force: true }); + }); +}); diff --git a/packages/@nitpicker/query/src/get-summary.ts b/packages/@nitpicker/query/src/get-summary.ts index 6cb63005..e0c2b5e8 100644 --- a/packages/@nitpicker/query/src/get-summary.ts +++ b/packages/@nitpicker/query/src/get-summary.ts @@ -67,8 +67,8 @@ export async function getSummary(accessor: ArchiveAccessor): Promise { }); }); - it('reads baseUrl/roots from the archive config, not from the read model', async () => { + it('reads baseUrl/roots/exclude settings from the archive config, not from the read model', async () => { const result = await getViewerSummary(archive); expect(result.baseUrl).toBe('https://example.com'); expect(result.roots).toEqual(['https://example.com']); + expect(result.excludes).toEqual([]); + expect(result.excludeKeywords).toEqual([]); + expect(result.excludeUrls).toEqual([]); + expect(result.maxExcludedDepth).toBe(0); }); }); @@ -363,4 +367,43 @@ describe('getViewerSummary', () => { ]); }); }); + + describe('with non-empty exclude settings (issue #261)', () => { + const workingDir = path.resolve( + __dirname, + '__test_fixtures_get_viewer_summary_excludes__', + ); + const archiveFilePath = path.resolve(workingDir, 'excludes-test.nitpicker'); + let archive: InstanceType; + + beforeAll(async () => { + const { mkdirSync } = await import('node:fs'); + mkdirSync(workingDir, { recursive: true }); + archive = await Archive.create({ filePath: archiveFilePath, cwd: workingDir }); + await archive.setConfig({ + ...BASE_CONFIG, + excludes: ['/admin/*'], + excludeKeywords: ['draft'], + excludeUrls: ['https://example.com/temp'], + maxExcludedDepth: 3, + }); + await buildViewerReadModel(archive); + }); + + afterAll(async () => { + if (archive) { + await archive.releaseHandle(); + } + const { rmSync } = await import('node:fs'); + rmSync(workingDir, { recursive: true, force: true }); + }); + + it('surfaces non-empty exclude settings from config, not hardcoded defaults', async () => { + const result = await getViewerSummary(archive); + expect(result.excludes).toEqual(['/admin/*']); + expect(result.excludeKeywords).toEqual(['draft']); + expect(result.excludeUrls).toEqual(['https://example.com/temp']); + expect(result.maxExcludedDepth).toBe(3); + }); + }); }); diff --git a/packages/@nitpicker/query/src/get-viewer-summary.ts b/packages/@nitpicker/query/src/get-viewer-summary.ts index f479a9be..34da2b31 100644 --- a/packages/@nitpicker/query/src/get-viewer-summary.ts +++ b/packages/@nitpicker/query/src/get-viewer-summary.ts @@ -12,9 +12,11 @@ import type { ArchiveAccessor } from '@nitpicker/crawler'; * `JSON.parse` calls, replacing `getSummary`'s multi-second full-table * aggregation on archives where the read model is current. * - * `baseUrl`/`roots` are not stored in `viewer_summary` (they come from - * `accessor.getConfig()`, independent of `pages` aggregation and already - * cheap) — this function merges them back in at read time. + * `baseUrl`/`roots`/exclude settings (`excludes`, `excludeKeywords`, + * `excludeUrls`, `maxExcludedDepth`) are not stored in `viewer_summary` + * (they come from `accessor.getConfig()`, independent of `pages` + * aggregation and already cheap) — this function merges them back in at + * read time. * * Callers are responsible for guarding with `isViewerReadModelCurrent` * first, the same convention `listViewerPages` uses — this function does @@ -44,6 +46,10 @@ export async function getViewerSummary( return { baseUrl: config.baseUrl, roots: config.roots, + excludes: config.excludes, + excludeKeywords: config.excludeKeywords, + excludeUrls: config.excludeUrls, + maxExcludedDepth: config.maxExcludedDepth, totalPages: Number(row.total_pages), internalPages: Number(row.internal_pages), externalPages: Number(row.external_pages), diff --git a/packages/@nitpicker/query/src/types.ts b/packages/@nitpicker/query/src/types.ts index ca7b4001..6b4c793f 100644 --- a/packages/@nitpicker/query/src/types.ts +++ b/packages/@nitpicker/query/src/types.ts @@ -553,6 +553,14 @@ export interface SummaryResult { baseUrl: string; /** All user-provided root URLs. Single-root archives report `[baseUrl]`. */ roots: string[]; + /** Maximum directory depth for excluded paths. */ + maxExcludedDepth: number; + /** Keywords used to exclude pages from crawling. */ + excludeKeywords: string[]; + /** URL patterns to exclude from crawling. */ + excludes: string[]; + /** URL prefixes to exclude from crawling. */ + excludeUrls: string[]; /** * Total number of HTML pages (internal + external) — the historical * "pages" count, restricted to `contentType IS NULL OR text/html` so diff --git a/packages/@nitpicker/viewer/src/create-app.spec.ts b/packages/@nitpicker/viewer/src/create-app.spec.ts index 71571892..7f5b3a51 100644 --- a/packages/@nitpicker/viewer/src/create-app.spec.ts +++ b/packages/@nitpicker/viewer/src/create-app.spec.ts @@ -33,10 +33,10 @@ describe('createApp', () => { fetchExternal: false, parallels: 1, roots: ['https://example.com'], - excludes: [], - excludeKeywords: [], - excludeUrls: [], - maxExcludedDepth: 0, + excludes: ['/admin/*'], + excludeKeywords: ['draft'], + excludeUrls: ['https://example.com/temp'], + maxExcludedDepth: 3, retry: 3, fromList: false, disableQueries: false, @@ -180,6 +180,10 @@ describe('createApp', () => { externalPages: number; internalContents: number; externalContents: number; + excludes: string[]; + excludeKeywords: string[]; + excludeUrls: string[]; + maxExcludedDepth: number; }; expect(body.totalPages).toBeGreaterThanOrEqual(2); /* `internalContents`/`externalContents` must pass through the API @@ -190,6 +194,14 @@ describe('createApp', () => { documented in the SummaryResult JSDoc. */ expect(body.internalContents).toBeGreaterThanOrEqual(body.internalPages); expect(body.externalContents).toBeGreaterThanOrEqual(body.externalPages); + /* End-to-end coverage for issue #261: exclude settings written via + `archive.setConfig()` (crawler package) must survive the full + config → getSummaryFastPath (query package) → HTTP JSON + (viewer package) pipeline, not just the query-layer unit tests. */ + expect(body.excludes).toEqual(['/admin/*']); + expect(body.excludeKeywords).toEqual(['draft']); + expect(body.excludeUrls).toEqual(['https://example.com/temp']); + expect(body.maxExcludedDepth).toBe(3); }); it('GET /api/pages はページ一覧を返す', async () => { diff --git a/packages/@nitpicker/viewer/src/precomputed-disk-cache.spec.ts b/packages/@nitpicker/viewer/src/precomputed-disk-cache.spec.ts index c4e884dc..a1d9f29a 100644 --- a/packages/@nitpicker/viewer/src/precomputed-disk-cache.spec.ts +++ b/packages/@nitpicker/viewer/src/precomputed-disk-cache.spec.ts @@ -127,6 +127,32 @@ describe('getOrComputeOnDisk', () => { expect(onDisk).toBe(siblingArtefact); }); + it('regenerates the artefact when a cache hit fails the isValid shape guard', async () => { + // Simulates a cache file written by an older nitpicker build whose + // `compute()` shape has since grown new required fields — the + // content-hash cache key doesn't change on a version upgrade, so + // the stale shape must be treated like corruption, not a hit. + const cacheDir = path.join(baseDir, 'stale-shape'); + const precomputedDir = path.join(cacheDir, 'precomputed'); + await fs.mkdir(precomputedDir, { recursive: true }); + await fs.writeFile( + path.join(precomputedDir, 'shaped.json'), + JSON.stringify({ old: true }), + ); + + const compute = vi.fn().mockResolvedValueOnce({ old: true, fresh: true }); + const result = await getOrComputeOnDisk( + cacheDir, + 'shaped', + compute, + (value: { fresh?: boolean }) => value.fresh === true, + ); + expect(result).toEqual({ old: true, fresh: true }); + expect(compute).toHaveBeenCalledTimes(1); + const onDisk = await fs.readFile(path.join(precomputedDir, 'shaped.json'), 'utf8'); + expect(JSON.parse(onDisk)).toEqual({ old: true, fresh: true }); + }); + it('reconstructs round-tripped Map shapes when callers serialise via entries arrays', async () => { // Map is not natively JSON-serialisable; callers that cache a // Map serialise it as [[k,v],...] entries. Verify the disk layer diff --git a/packages/@nitpicker/viewer/src/precomputed-disk-cache.ts b/packages/@nitpicker/viewer/src/precomputed-disk-cache.ts index 8955dab9..4a800043 100644 --- a/packages/@nitpicker/viewer/src/precomputed-disk-cache.ts +++ b/packages/@nitpicker/viewer/src/precomputed-disk-cache.ts @@ -48,12 +48,21 @@ const PRECOMPUTED_DIR_NAME = 'precomputed'; * `"isolated-clusters"`). Used as the on-disk filename. * @param compute - Loader invoked on cache miss. Its return value is * what gets persisted; subsequent reads return the parsed JSON. + * @param isValid - Optional shape guard run against a cache hit before + * it is returned. A cached artefact from an older nitpicker build can + * be missing fields a newer `compute()` shape requires (the archive's + * content-hash key only invalidates on archive mutation, not on a + * nitpicker version upgrade) — returning it as-is would hand callers + * an object that doesn't match `T` at runtime. When `isValid` returns + * `false` the hit is treated exactly like corrupt JSON: fall through + * to `compute()` and overwrite. * @returns The cached or freshly-computed artefact. */ export async function getOrComputeOnDisk( cacheDir: string, name: string, compute: () => Promise, + isValid?: (value: T) => boolean, ): Promise { const dir = path.join(cacheDir, PRECOMPUTED_DIR_NAME); const file = path.join(dir, `${name}.json`); @@ -67,16 +76,20 @@ export async function getOrComputeOnDisk( let isCorrupt = false; try { const raw = await fs.readFile(file, 'utf8'); - return JSON.parse(raw) as T; + const parsed = JSON.parse(raw) as T; + if (isValid && !isValid(parsed)) { + throw new Error('stale cache shape'); + } + return parsed; } catch (error) { const code = (error as NodeJS.ErrnoException).code; if (code !== 'ENOENT') { // Either an ENOENT we did not anticipate (handled below by - // treating as miss) or a parse error — both fall through - // to compute. We only flag corruption when the file was - // reachable but the JSON did not parse, so the post-compute - // race-check overwrites the bad data. - isCorrupt = !code; // SyntaxError has no .code + // treating as miss), a parse error, or a failed `isValid` + // check — all fall through to compute. We only flag + // corruption when the file was reachable but unusable, so + // the post-compute race-check overwrites the bad data. + isCorrupt = !code; // SyntaxError / isValid failure has no .code } } diff --git a/packages/@nitpicker/viewer/src/summary-cache.spec.ts b/packages/@nitpicker/viewer/src/summary-cache.spec.ts index bf1197f4..c71aa48d 100644 --- a/packages/@nitpicker/viewer/src/summary-cache.spec.ts +++ b/packages/@nitpicker/viewer/src/summary-cache.spec.ts @@ -24,6 +24,7 @@ vi.mock('./precomputed-disk-cache.js', () => ({ })); const { getSummary, getSummaryFastPath } = await import('@nitpicker/query'); +const { getOrComputeOnDisk } = await import('./precomputed-disk-cache.js'); afterEach(() => { vi.clearAllMocks(); @@ -60,6 +61,10 @@ function makeSummary(baseUrl: string): SummaryResult { return { baseUrl, roots: [], + excludes: [], + excludeKeywords: [], + excludeUrls: [], + maxExcludedDepth: 0, totalPages: 0, internalPages: 0, externalPages: 0, @@ -75,6 +80,8 @@ function makeSummary(baseUrl: string): SummaryResult { ogImage: 0, }, contentTypeDistribution: [], + networkOutageAffectedFailures: 0, + consoleLogCounts: { pageerror: 0, error: 0, warn: 0 }, }; } @@ -165,4 +172,31 @@ describe('getCachedSummary', () => { expect(recovered.baseUrl).toBe('recovered'); expect(getSummaryFastPath).toHaveBeenCalledTimes(2); }); + + it('passes an isValid guard to getOrComputeOnDisk that rejects a disk cache missing exclude-setting fields (issue #261)', async () => { + // The disk cache's content-hash key does not change on a nitpicker + // version upgrade, so a `summary.json` written before issue #261 + // could otherwise be replayed as-is and crash the Summary view's + // `data.excludes.length` read. Verify the guard summary-cache.ts + // wires into getOrComputeOnDisk actually enforces the new shape. + vi.mocked(getSummaryFastPath).mockResolvedValueOnce(makeSummary('guarded')); + const context = makeContext('archive_guard'); + + await getCachedSummary(context); + + const isValid = vi.mocked(getOrComputeOnDisk).mock.calls[0]?.[3] as + | ((value: SummaryResult) => boolean) + | undefined; + expect(isValid).toBeInstanceOf(Function); + expect(isValid?.(makeSummary('complete'))).toBe(true); + expect( + isValid?.({ ...makeSummary('stale'), excludes: undefined } as SummaryResult), + ).toBe(false); + expect( + isValid?.({ + ...makeSummary('stale'), + maxExcludedDepth: undefined, + } as SummaryResult), + ).toBe(false); + }); }); diff --git a/packages/@nitpicker/viewer/src/summary-cache.ts b/packages/@nitpicker/viewer/src/summary-cache.ts index 490ce394..202ced63 100644 --- a/packages/@nitpicker/viewer/src/summary-cache.ts +++ b/packages/@nitpicker/viewer/src/summary-cache.ts @@ -74,8 +74,31 @@ export async function getCachedSummary(context: ArchiveContext): Promise { const accessor = context.manager.get(context.archiveId); - return getOrComputeOnDisk(accessor.tmpDir, 'summary', () => - getSummaryFastPath(accessor), + return getOrComputeOnDisk( + accessor.tmpDir, + 'summary', + () => getSummaryFastPath(accessor), + isCompleteSummaryResult, ); }); } + +/** + * Guards against a disk-cached `summary.json` written by a nitpicker + * build that predates the exclude-settings fields (issue #261) — the + * archive's content-hash cache key does not change on a nitpicker + * version upgrade, so an old-shaped artefact would otherwise be + * returned as-is and crash `summary-view.tsx`'s `data.excludes.length` + * read. + * @param value - A parsed disk-cache hit to validate. + * @returns Whether `value` has every exclude-settings field the current + * `SummaryResult` shape requires. + */ +function isCompleteSummaryResult(value: SummaryResult): boolean { + return ( + Array.isArray(value.excludes) && + Array.isArray(value.excludeKeywords) && + Array.isArray(value.excludeUrls) && + typeof value.maxExcludedDepth === 'number' + ); +} diff --git a/packages/@nitpicker/viewer/web/i18n/translations.ts b/packages/@nitpicker/viewer/web/i18n/translations.ts index dcb68823..08716122 100644 --- a/packages/@nitpicker/viewer/web/i18n/translations.ts +++ b/packages/@nitpicker/viewer/web/i18n/translations.ts @@ -92,6 +92,10 @@ export const translations: Record> = { title: 'Summary', description: 'Site-wide overview: page counts, HTTP status distribution, and metadata fulfillment.', + excludes: 'Excludes', + excludeKeywords: 'Exclude keywords', + excludeUrls: 'Exclude URLs', + maxExcludedDepth: 'Max excluded depth', internalContents: 'Internal contents (all)', internalPages: 'Internal pages (HTML)', externalContents: 'External contents (all)', @@ -531,6 +535,10 @@ export const translations: Record> = { title: 'サマリー', description: 'サイト全体の概要:ページ数、HTTP ステータス分布、メタデータの充足率を表示します。', + excludes: '除外パターン', + excludeKeywords: '除外キーワード', + excludeUrls: '除外URL', + maxExcludedDepth: '除外の最大深度', internalContents: '総コンテンツ数(内部)', internalPages: '総ページ数(内部)', externalContents: '外部リンク(外部コンテンツ)', diff --git a/packages/@nitpicker/viewer/web/routes/summary-view.tsx b/packages/@nitpicker/viewer/web/routes/summary-view.tsx index 7da07591..565741db 100644 --- a/packages/@nitpicker/viewer/web/routes/summary-view.tsx +++ b/packages/@nitpicker/viewer/web/routes/summary-view.tsx @@ -118,6 +118,26 @@ export function SummaryView() { {root}

))} + {/* Exclude settings, same

row style as roots above. Each row is + suppressed when its value is empty/zero — most archives crawl + without exclusions, so an always-shown block would be noise the + same way the console-log cards below are gated on non-zero. */} + {[ + { key: 'excludes', text: data.excludes.join(', ') || null }, + { key: 'excludeKeywords', text: data.excludeKeywords.join(', ') || null }, + { key: 'excludeUrls', text: data.excludeUrls.join(', ') || null }, + { + key: 'maxExcludedDepth', + text: data.maxExcludedDepth > 0 ? String(data.maxExcludedDepth) : null, + }, + ].map( + (row) => + row.text !== null && ( +

+ {t(`views.summary.${row.key}`)}: {row.text} +

+ ), + )} {/* Three cards (was four). "Roots" is dropped because the root URL list is already rendered above as `

` rows — a count card is redundant. The remaining three give the user the three numbers