diff --git a/packages/@nitpicker/crawler/src/archive/database.spec.ts b/packages/@nitpicker/crawler/src/archive/database.spec.ts
index 954ca6b0..450ffe83 100644
--- a/packages/@nitpicker/crawler/src/archive/database.spec.ts
+++ b/packages/@nitpicker/crawler/src/archive/database.spec.ts
@@ -3443,6 +3443,47 @@ describe('getJSON (getConfig 経由)', () => {
await db2.destroy();
});
+
+ it('info.maxExcludedDepth が NULL の場合フォールバック値 0 を返す(issue #261)', async () => {
+ // `maxExcludedDepth` is a plain integer column, not a JSON column
+ // like `excludes`, so it needs its own NULL-fallback coverage —
+ // getJSON's fallback path above does not exercise it.
+ const nullDepthDbPath = path.resolve(
+ workingDir,
+ 'null-max-excluded-depth-test.sqlite',
+ );
+ const db = await Database.connect({ filename: nullDepthDbPath });
+
+ const config: Config = {
+ version: '0.13.0',
+ name: 'test',
+ baseUrl: 'https://example.com',
+ roots: ['https://example.com'],
+ recursive: false,
+ interval: 500,
+ image: false,
+ fetchExternal: false,
+ parallels: 1,
+ excludes: [],
+ excludeKeywords: [],
+ excludeUrls: [],
+ maxExcludedDepth: 3,
+ retry: 3,
+ fromList: false,
+ disableQueries: false,
+ userAgent: 'test',
+ ignoreRobots: false,
+ };
+
+ await db.setConfig(config);
+ await db.getKnex()('info').update({ maxExcludedDepth: null });
+
+ const retrieved = await db.getConfig();
+ expect(retrieved.maxExcludedDepth).toBe(0);
+
+ await db.destroy();
+ await remove(nullDepthDbPath);
+ });
});
describe('insertPageError', () => {
diff --git a/packages/@nitpicker/crawler/src/archive/db-ops/config/get-config.ts b/packages/@nitpicker/crawler/src/archive/db-ops/config/get-config.ts
index 73b32315..8f11a6c3 100644
--- a/packages/@nitpicker/crawler/src/archive/db-ops/config/get-config.ts
+++ b/packages/@nitpicker/crawler/src/archive/db-ops/config/get-config.ts
@@ -23,6 +23,7 @@ export async function getConfig(knex: Knex): Promise {
excludeUrls: getJSON(config.excludeUrls, []),
roots: getJSON(config.roots, []),
retry: config.retry ?? 3,
+ maxExcludedDepth: config.maxExcludedDepth ?? 0,
};
// @ts-expect-error — `id` is the primary key, not part of the public Config shape
delete opt.id;
diff --git a/packages/@nitpicker/query/src/get-summary-fast-path.spec.ts b/packages/@nitpicker/query/src/get-summary-fast-path.spec.ts
index daa766ec..1ba2bd4f 100644
--- a/packages/@nitpicker/query/src/get-summary-fast-path.spec.ts
+++ b/packages/@nitpicker/query/src/get-summary-fast-path.spec.ts
@@ -25,6 +25,10 @@ function makeSummary(baseUrl: string): SummaryResult {
return {
baseUrl,
roots: [],
+ excludes: [],
+ excludeKeywords: [],
+ excludeUrls: [],
+ maxExcludedDepth: 0,
totalPages: 0,
internalPages: 0,
externalPages: 0,
@@ -40,6 +44,8 @@ function makeSummary(baseUrl: string): SummaryResult {
ogImage: 0,
},
contentTypeDistribution: [],
+ networkOutageAffectedFailures: 0,
+ consoleLogCounts: { pageerror: 0, error: 0, warn: 0 },
};
}
diff --git a/packages/@nitpicker/query/src/get-summary.spec.ts b/packages/@nitpicker/query/src/get-summary.spec.ts
index ec96df5d..2b6548a0 100644
--- a/packages/@nitpicker/query/src/get-summary.spec.ts
+++ b/packages/@nitpicker/query/src/get-summary.spec.ts
@@ -1106,3 +1106,42 @@ describe('getSummary: console log counts (issue #228)', () => {
rmSync(dir, { recursive: true, force: true });
});
});
+
+describe('getSummary: exclude settings (issue #261)', () => {
+ it('passes through excludes/excludeKeywords/excludeUrls/maxExcludedDepth from config', async () => {
+ const dir = path.resolve(__dirname, '__test_fixtures_summary_excludes__');
+ const archiveFilePath = path.resolve(dir, 'summary-excludes.nitpicker');
+ const { mkdirSync, rmSync } = await import('node:fs');
+ mkdirSync(dir, { recursive: true });
+ const archive = await Archive.create({ filePath: archiveFilePath, cwd: dir });
+ await archive.setConfig({
+ baseUrl: 'https://example.com',
+ roots: ['https://example.com'],
+ name: 'test',
+ version: '0.13.0',
+ recursive: true,
+ interval: 0,
+ image: true,
+ fetchExternal: false,
+ parallels: 1,
+ excludes: ['/admin/*'],
+ excludeKeywords: ['draft'],
+ excludeUrls: ['https://example.com/temp'],
+ maxExcludedDepth: 3,
+ retry: 3,
+ fromList: false,
+ disableQueries: false,
+ userAgent: 'test',
+ ignoreRobots: false,
+ });
+
+ const result = await getSummary(archive);
+ expect(result.excludes).toEqual(['/admin/*']);
+ expect(result.excludeKeywords).toEqual(['draft']);
+ expect(result.excludeUrls).toEqual(['https://example.com/temp']);
+ expect(result.maxExcludedDepth).toBe(3);
+
+ await archive.close();
+ rmSync(dir, { recursive: true, force: true });
+ });
+});
diff --git a/packages/@nitpicker/query/src/get-summary.ts b/packages/@nitpicker/query/src/get-summary.ts
index 6cb63005..e0c2b5e8 100644
--- a/packages/@nitpicker/query/src/get-summary.ts
+++ b/packages/@nitpicker/query/src/get-summary.ts
@@ -67,8 +67,8 @@ export async function getSummary(accessor: ArchiveAccessor): Promise {
});
});
- it('reads baseUrl/roots from the archive config, not from the read model', async () => {
+ it('reads baseUrl/roots/exclude settings from the archive config, not from the read model', async () => {
const result = await getViewerSummary(archive);
expect(result.baseUrl).toBe('https://example.com');
expect(result.roots).toEqual(['https://example.com']);
+ expect(result.excludes).toEqual([]);
+ expect(result.excludeKeywords).toEqual([]);
+ expect(result.excludeUrls).toEqual([]);
+ expect(result.maxExcludedDepth).toBe(0);
});
});
@@ -363,4 +367,43 @@ describe('getViewerSummary', () => {
]);
});
});
+
+ describe('with non-empty exclude settings (issue #261)', () => {
+ const workingDir = path.resolve(
+ __dirname,
+ '__test_fixtures_get_viewer_summary_excludes__',
+ );
+ const archiveFilePath = path.resolve(workingDir, 'excludes-test.nitpicker');
+ let archive: InstanceType;
+
+ beforeAll(async () => {
+ const { mkdirSync } = await import('node:fs');
+ mkdirSync(workingDir, { recursive: true });
+ archive = await Archive.create({ filePath: archiveFilePath, cwd: workingDir });
+ await archive.setConfig({
+ ...BASE_CONFIG,
+ excludes: ['/admin/*'],
+ excludeKeywords: ['draft'],
+ excludeUrls: ['https://example.com/temp'],
+ maxExcludedDepth: 3,
+ });
+ await buildViewerReadModel(archive);
+ });
+
+ afterAll(async () => {
+ if (archive) {
+ await archive.releaseHandle();
+ }
+ const { rmSync } = await import('node:fs');
+ rmSync(workingDir, { recursive: true, force: true });
+ });
+
+ it('surfaces non-empty exclude settings from config, not hardcoded defaults', async () => {
+ const result = await getViewerSummary(archive);
+ expect(result.excludes).toEqual(['/admin/*']);
+ expect(result.excludeKeywords).toEqual(['draft']);
+ expect(result.excludeUrls).toEqual(['https://example.com/temp']);
+ expect(result.maxExcludedDepth).toBe(3);
+ });
+ });
});
diff --git a/packages/@nitpicker/query/src/get-viewer-summary.ts b/packages/@nitpicker/query/src/get-viewer-summary.ts
index f479a9be..34da2b31 100644
--- a/packages/@nitpicker/query/src/get-viewer-summary.ts
+++ b/packages/@nitpicker/query/src/get-viewer-summary.ts
@@ -12,9 +12,11 @@ import type { ArchiveAccessor } from '@nitpicker/crawler';
* `JSON.parse` calls, replacing `getSummary`'s multi-second full-table
* aggregation on archives where the read model is current.
*
- * `baseUrl`/`roots` are not stored in `viewer_summary` (they come from
- * `accessor.getConfig()`, independent of `pages` aggregation and already
- * cheap) — this function merges them back in at read time.
+ * `baseUrl`/`roots`/exclude settings (`excludes`, `excludeKeywords`,
+ * `excludeUrls`, `maxExcludedDepth`) are not stored in `viewer_summary`
+ * (they come from `accessor.getConfig()`, independent of `pages`
+ * aggregation and already cheap) — this function merges them back in at
+ * read time.
*
* Callers are responsible for guarding with `isViewerReadModelCurrent`
* first, the same convention `listViewerPages` uses — this function does
@@ -44,6 +46,10 @@ export async function getViewerSummary(
return {
baseUrl: config.baseUrl,
roots: config.roots,
+ excludes: config.excludes,
+ excludeKeywords: config.excludeKeywords,
+ excludeUrls: config.excludeUrls,
+ maxExcludedDepth: config.maxExcludedDepth,
totalPages: Number(row.total_pages),
internalPages: Number(row.internal_pages),
externalPages: Number(row.external_pages),
diff --git a/packages/@nitpicker/query/src/types.ts b/packages/@nitpicker/query/src/types.ts
index ca7b4001..6b4c793f 100644
--- a/packages/@nitpicker/query/src/types.ts
+++ b/packages/@nitpicker/query/src/types.ts
@@ -553,6 +553,14 @@ export interface SummaryResult {
baseUrl: string;
/** All user-provided root URLs. Single-root archives report `[baseUrl]`. */
roots: string[];
+ /** Maximum directory depth for excluded paths. */
+ maxExcludedDepth: number;
+ /** Keywords used to exclude pages from crawling. */
+ excludeKeywords: string[];
+ /** URL patterns to exclude from crawling. */
+ excludes: string[];
+ /** URL prefixes to exclude from crawling. */
+ excludeUrls: string[];
/**
* Total number of HTML pages (internal + external) — the historical
* "pages" count, restricted to `contentType IS NULL OR text/html` so
diff --git a/packages/@nitpicker/viewer/src/create-app.spec.ts b/packages/@nitpicker/viewer/src/create-app.spec.ts
index 71571892..7f5b3a51 100644
--- a/packages/@nitpicker/viewer/src/create-app.spec.ts
+++ b/packages/@nitpicker/viewer/src/create-app.spec.ts
@@ -33,10 +33,10 @@ describe('createApp', () => {
fetchExternal: false,
parallels: 1,
roots: ['https://example.com'],
- excludes: [],
- excludeKeywords: [],
- excludeUrls: [],
- maxExcludedDepth: 0,
+ excludes: ['/admin/*'],
+ excludeKeywords: ['draft'],
+ excludeUrls: ['https://example.com/temp'],
+ maxExcludedDepth: 3,
retry: 3,
fromList: false,
disableQueries: false,
@@ -180,6 +180,10 @@ describe('createApp', () => {
externalPages: number;
internalContents: number;
externalContents: number;
+ excludes: string[];
+ excludeKeywords: string[];
+ excludeUrls: string[];
+ maxExcludedDepth: number;
};
expect(body.totalPages).toBeGreaterThanOrEqual(2);
/* `internalContents`/`externalContents` must pass through the API
@@ -190,6 +194,14 @@ describe('createApp', () => {
documented in the SummaryResult JSDoc. */
expect(body.internalContents).toBeGreaterThanOrEqual(body.internalPages);
expect(body.externalContents).toBeGreaterThanOrEqual(body.externalPages);
+ /* End-to-end coverage for issue #261: exclude settings written via
+ `archive.setConfig()` (crawler package) must survive the full
+ config → getSummaryFastPath (query package) → HTTP JSON
+ (viewer package) pipeline, not just the query-layer unit tests. */
+ expect(body.excludes).toEqual(['/admin/*']);
+ expect(body.excludeKeywords).toEqual(['draft']);
+ expect(body.excludeUrls).toEqual(['https://example.com/temp']);
+ expect(body.maxExcludedDepth).toBe(3);
});
it('GET /api/pages はページ一覧を返す', async () => {
diff --git a/packages/@nitpicker/viewer/src/precomputed-disk-cache.spec.ts b/packages/@nitpicker/viewer/src/precomputed-disk-cache.spec.ts
index c4e884dc..a1d9f29a 100644
--- a/packages/@nitpicker/viewer/src/precomputed-disk-cache.spec.ts
+++ b/packages/@nitpicker/viewer/src/precomputed-disk-cache.spec.ts
@@ -127,6 +127,32 @@ describe('getOrComputeOnDisk', () => {
expect(onDisk).toBe(siblingArtefact);
});
+ it('regenerates the artefact when a cache hit fails the isValid shape guard', async () => {
+ // Simulates a cache file written by an older nitpicker build whose
+ // `compute()` shape has since grown new required fields — the
+ // content-hash cache key doesn't change on a version upgrade, so
+ // the stale shape must be treated like corruption, not a hit.
+ const cacheDir = path.join(baseDir, 'stale-shape');
+ const precomputedDir = path.join(cacheDir, 'precomputed');
+ await fs.mkdir(precomputedDir, { recursive: true });
+ await fs.writeFile(
+ path.join(precomputedDir, 'shaped.json'),
+ JSON.stringify({ old: true }),
+ );
+
+ const compute = vi.fn().mockResolvedValueOnce({ old: true, fresh: true });
+ const result = await getOrComputeOnDisk(
+ cacheDir,
+ 'shaped',
+ compute,
+ (value: { fresh?: boolean }) => value.fresh === true,
+ );
+ expect(result).toEqual({ old: true, fresh: true });
+ expect(compute).toHaveBeenCalledTimes(1);
+ const onDisk = await fs.readFile(path.join(precomputedDir, 'shaped.json'), 'utf8');
+ expect(JSON.parse(onDisk)).toEqual({ old: true, fresh: true });
+ });
+
it('reconstructs round-tripped Map shapes when callers serialise via entries arrays', async () => {
// Map is not natively JSON-serialisable; callers that cache a
// Map serialise it as [[k,v],...] entries. Verify the disk layer
diff --git a/packages/@nitpicker/viewer/src/precomputed-disk-cache.ts b/packages/@nitpicker/viewer/src/precomputed-disk-cache.ts
index 8955dab9..4a800043 100644
--- a/packages/@nitpicker/viewer/src/precomputed-disk-cache.ts
+++ b/packages/@nitpicker/viewer/src/precomputed-disk-cache.ts
@@ -48,12 +48,21 @@ const PRECOMPUTED_DIR_NAME = 'precomputed';
* `"isolated-clusters"`). Used as the on-disk filename.
* @param compute - Loader invoked on cache miss. Its return value is
* what gets persisted; subsequent reads return the parsed JSON.
+ * @param isValid - Optional shape guard run against a cache hit before
+ * it is returned. A cached artefact from an older nitpicker build can
+ * be missing fields a newer `compute()` shape requires (the archive's
+ * content-hash key only invalidates on archive mutation, not on a
+ * nitpicker version upgrade) — returning it as-is would hand callers
+ * an object that doesn't match `T` at runtime. When `isValid` returns
+ * `false` the hit is treated exactly like corrupt JSON: fall through
+ * to `compute()` and overwrite.
* @returns The cached or freshly-computed artefact.
*/
export async function getOrComputeOnDisk(
cacheDir: string,
name: string,
compute: () => Promise,
+ isValid?: (value: T) => boolean,
): Promise {
const dir = path.join(cacheDir, PRECOMPUTED_DIR_NAME);
const file = path.join(dir, `${name}.json`);
@@ -67,16 +76,20 @@ export async function getOrComputeOnDisk(
let isCorrupt = false;
try {
const raw = await fs.readFile(file, 'utf8');
- return JSON.parse(raw) as T;
+ const parsed = JSON.parse(raw) as T;
+ if (isValid && !isValid(parsed)) {
+ throw new Error('stale cache shape');
+ }
+ return parsed;
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code !== 'ENOENT') {
// Either an ENOENT we did not anticipate (handled below by
- // treating as miss) or a parse error — both fall through
- // to compute. We only flag corruption when the file was
- // reachable but the JSON did not parse, so the post-compute
- // race-check overwrites the bad data.
- isCorrupt = !code; // SyntaxError has no .code
+ // treating as miss), a parse error, or a failed `isValid`
+ // check — all fall through to compute. We only flag
+ // corruption when the file was reachable but unusable, so
+ // the post-compute race-check overwrites the bad data.
+ isCorrupt = !code; // SyntaxError / isValid failure has no .code
}
}
diff --git a/packages/@nitpicker/viewer/src/summary-cache.spec.ts b/packages/@nitpicker/viewer/src/summary-cache.spec.ts
index bf1197f4..c71aa48d 100644
--- a/packages/@nitpicker/viewer/src/summary-cache.spec.ts
+++ b/packages/@nitpicker/viewer/src/summary-cache.spec.ts
@@ -24,6 +24,7 @@ vi.mock('./precomputed-disk-cache.js', () => ({
}));
const { getSummary, getSummaryFastPath } = await import('@nitpicker/query');
+const { getOrComputeOnDisk } = await import('./precomputed-disk-cache.js');
afterEach(() => {
vi.clearAllMocks();
@@ -60,6 +61,10 @@ function makeSummary(baseUrl: string): SummaryResult {
return {
baseUrl,
roots: [],
+ excludes: [],
+ excludeKeywords: [],
+ excludeUrls: [],
+ maxExcludedDepth: 0,
totalPages: 0,
internalPages: 0,
externalPages: 0,
@@ -75,6 +80,8 @@ function makeSummary(baseUrl: string): SummaryResult {
ogImage: 0,
},
contentTypeDistribution: [],
+ networkOutageAffectedFailures: 0,
+ consoleLogCounts: { pageerror: 0, error: 0, warn: 0 },
};
}
@@ -165,4 +172,31 @@ describe('getCachedSummary', () => {
expect(recovered.baseUrl).toBe('recovered');
expect(getSummaryFastPath).toHaveBeenCalledTimes(2);
});
+
+ it('passes an isValid guard to getOrComputeOnDisk that rejects a disk cache missing exclude-setting fields (issue #261)', async () => {
+ // The disk cache's content-hash key does not change on a nitpicker
+ // version upgrade, so a `summary.json` written before issue #261
+ // could otherwise be replayed as-is and crash the Summary view's
+ // `data.excludes.length` read. Verify the guard summary-cache.ts
+ // wires into getOrComputeOnDisk actually enforces the new shape.
+ vi.mocked(getSummaryFastPath).mockResolvedValueOnce(makeSummary('guarded'));
+ const context = makeContext('archive_guard');
+
+ await getCachedSummary(context);
+
+ const isValid = vi.mocked(getOrComputeOnDisk).mock.calls[0]?.[3] as
+ | ((value: SummaryResult) => boolean)
+ | undefined;
+ expect(isValid).toBeInstanceOf(Function);
+ expect(isValid?.(makeSummary('complete'))).toBe(true);
+ expect(
+ isValid?.({ ...makeSummary('stale'), excludes: undefined } as SummaryResult),
+ ).toBe(false);
+ expect(
+ isValid?.({
+ ...makeSummary('stale'),
+ maxExcludedDepth: undefined,
+ } as SummaryResult),
+ ).toBe(false);
+ });
});
diff --git a/packages/@nitpicker/viewer/src/summary-cache.ts b/packages/@nitpicker/viewer/src/summary-cache.ts
index 490ce394..202ced63 100644
--- a/packages/@nitpicker/viewer/src/summary-cache.ts
+++ b/packages/@nitpicker/viewer/src/summary-cache.ts
@@ -74,8 +74,31 @@ export async function getCachedSummary(context: ArchiveContext): Promise {
const accessor = context.manager.get(context.archiveId);
- return getOrComputeOnDisk(accessor.tmpDir, 'summary', () =>
- getSummaryFastPath(accessor),
+ return getOrComputeOnDisk(
+ accessor.tmpDir,
+ 'summary',
+ () => getSummaryFastPath(accessor),
+ isCompleteSummaryResult,
);
});
}
+
+/**
+ * Guards against a disk-cached `summary.json` written by a nitpicker
+ * build that predates the exclude-settings fields (issue #261) — the
+ * archive's content-hash cache key does not change on a nitpicker
+ * version upgrade, so an old-shaped artefact would otherwise be
+ * returned as-is and crash `summary-view.tsx`'s `data.excludes.length`
+ * read.
+ * @param value - A parsed disk-cache hit to validate.
+ * @returns Whether `value` has every exclude-settings field the current
+ * `SummaryResult` shape requires.
+ */
+function isCompleteSummaryResult(value: SummaryResult): boolean {
+ return (
+ Array.isArray(value.excludes) &&
+ Array.isArray(value.excludeKeywords) &&
+ Array.isArray(value.excludeUrls) &&
+ typeof value.maxExcludedDepth === 'number'
+ );
+}
diff --git a/packages/@nitpicker/viewer/web/i18n/translations.ts b/packages/@nitpicker/viewer/web/i18n/translations.ts
index dcb68823..08716122 100644
--- a/packages/@nitpicker/viewer/web/i18n/translations.ts
+++ b/packages/@nitpicker/viewer/web/i18n/translations.ts
@@ -92,6 +92,10 @@ export const translations: Record> = {
title: 'Summary',
description:
'Site-wide overview: page counts, HTTP status distribution, and metadata fulfillment.',
+ excludes: 'Excludes',
+ excludeKeywords: 'Exclude keywords',
+ excludeUrls: 'Exclude URLs',
+ maxExcludedDepth: 'Max excluded depth',
internalContents: 'Internal contents (all)',
internalPages: 'Internal pages (HTML)',
externalContents: 'External contents (all)',
@@ -531,6 +535,10 @@ export const translations: Record> = {
title: 'サマリー',
description:
'サイト全体の概要:ページ数、HTTP ステータス分布、メタデータの充足率を表示します。',
+ excludes: '除外パターン',
+ excludeKeywords: '除外キーワード',
+ excludeUrls: '除外URL',
+ maxExcludedDepth: '除外の最大深度',
internalContents: '総コンテンツ数(内部)',
internalPages: '総ページ数(内部)',
externalContents: '外部リンク(外部コンテンツ)',
diff --git a/packages/@nitpicker/viewer/web/routes/summary-view.tsx b/packages/@nitpicker/viewer/web/routes/summary-view.tsx
index 7da07591..565741db 100644
--- a/packages/@nitpicker/viewer/web/routes/summary-view.tsx
+++ b/packages/@nitpicker/viewer/web/routes/summary-view.tsx
@@ -118,6 +118,26 @@ export function SummaryView() {
{root}
))}
+ {/* Exclude settings, same row style as roots above. Each row is
+ suppressed when its value is empty/zero — most archives crawl
+ without exclusions, so an always-shown block would be noise the
+ same way the console-log cards below are gated on non-zero. */}
+ {[
+ { key: 'excludes', text: data.excludes.join(', ') || null },
+ { key: 'excludeKeywords', text: data.excludeKeywords.join(', ') || null },
+ { key: 'excludeUrls', text: data.excludeUrls.join(', ') || null },
+ {
+ key: 'maxExcludedDepth',
+ text: data.maxExcludedDepth > 0 ? String(data.maxExcludedDepth) : null,
+ },
+ ].map(
+ (row) =>
+ row.text !== null && (
+
+ {t(`views.summary.${row.key}`)}: {row.text}
+
+ ),
+ )}
{/* Three cards (was four). "Roots" is dropped because the root URL
list is already rendered above as `` rows — a count card is
redundant. The remaining three give the user the three numbers