Mark dedupe-cap trap pages post-hoc; default-enable --dedupe-cap

.github/workflows/e2e.yml

@@ -76,7 +76,11 @@ jobs:
     # flaky failures would require re-running anyway.
     strategy:
       fail-fast: false
-      # LPT-balanced as of 2026-06-26 measurement (1340 s total, target 335 s/shard).
+      # LPT-balanced as of 2026-06-26 measurement (1340 s total, target 335 s/shard);
+      # `dedupe-cap.e2e.ts` (a full crawl -> viewer-read-model-build -> query
+      # pipeline test) was placed on `append-pipeline` rather than
+      # `inventory-snapshot` since the latter was already the heaviest shard
+      # and adding it there pushed the job past the 15-minute timeout.
       # If a shard starts overshooting in the Actions UI, redo the bin-packing
       # by hand from a recent run's per-file timings.
       matrix:
@@ -105,7 +109,6 @@ jobs:
                 special-char-auth.e2e.ts,
                 empty-password-auth.e2e.ts,
                 main-contents.e2e.ts,
-                dedupe-cap.e2e.ts,
               ]
           - id: retry-exclude
             files:
@@ -131,6 +134,7 @@ jobs:
                 cli-process-exit.e2e.ts,
                 cli-version.e2e.ts,
                 viewer-migrated-archive.e2e.ts,
+                dedupe-cap.e2e.ts,
               ]
     steps:
       - uses: actions/checkout@v7.0.0

cspell.json

@@ -14,6 +14,9 @@
 		"Misra",
 		"Gries",
 
+		// SQLite PRAGMA table_info() column name
+		"dflt",
+
 		// nitpicker
 		"readtext",
 		"unanalyzed",

packages/@nitpicker/cli/src/commands/crawl-flag-parsing.spec.ts

@@ -91,4 +91,24 @@ describe('crawl CLI flag parsing (parseCli integration)', () => {
 		]);
 		expect(result.flags.interval).toBe(5000);
 	});
+
+	it('defaults dedupeCap to 10 when the flag is not supplied', () => {
+		const result = runParse(['https://example.com/']);
+		expect(result.flags.dedupeCap).toBe(10);
+	});
+
+	it('--no-dedupe-cap parses to 0, not undefined — the sentinel mapFlagsToCrawlConfig converts to null', () => {
+		// Regression guard for the exact gotcha `map-flags-to-crawl-config.ts`
+		// documents: yargs-parser's boolean-negation coercion turns
+		// `--no-dedupe-cap` into `Number(false) === 0`, never `undefined`. If
+		// this ever changed (e.g. a roar upgrade), `mapFlagsToCrawlConfig`'s
+		// `flags.dedupeCap || null` conversion would need to change with it.
+		const result = runParse(['https://example.com/', '--no-dedupe-cap']);
+		expect(result.flags.dedupeCap).toBe(0);
+	});
+
+	it('accepts an explicit --dedupeCap value, overriding the default', () => {
+		const result = runParse(['https://example.com/', '--dedupeCap', '25']);
+		expect(result.flags.dedupeCap).toBe(25);
+	});
 });

packages/@nitpicker/cli/src/commands/crawl.spec.ts

@@ -211,6 +211,11 @@ describe('startCrawl', () => {
 		);
 	});
 
+	it('dedupeCap フラグは default: 10 で on-by-default （--no-dedupe-cap / --dedupeCap 0 で無効化できる前提）', async () => {
+		const { commandDef } = await import('./crawl.js');
+		expect(commandDef.flags.dedupeCap.default).toBe(10);
+	});
+
 	it('--list モードでも recursive: false になる', async () => {
 		const { startCrawl } = await import('./crawl.js');
 		await startCrawl(

packages/@nitpicker/cli/src/commands/crawl.ts

@@ -129,13 +129,14 @@ export const commandDef = {
 		},
 		dedupeCap: {
 			type: 'number',
+			default: 10,
 			group: 'Scope & filtering',
-			desc: 'Same-cluster soft cap: stop enqueueing newly-discovered internal URLs whose shape (e.g. `/news/date/{n}/`) has accumulated this many matching-title/description/og-tag observations. Opt-in — omit to disable. Backstop against a site that keeps serving 2xx for a self-generating pager/query-parameter trap; see `query dedupe-cap-events` for what fired.',
+			desc: 'Same-cluster soft cap: stop enqueueing newly-discovered internal URLs whose shape (e.g. `/news/date/{n}/`) has accumulated this many matching-title/description/og-tag observations. On by default (10) as a backstop against a site that keeps serving 2xx for a self-generating pager/query-parameter trap — false positives on legitimate large sections are structurally prevented (each such page differs in title/og tags, so the majority-vote counter never accumulates). Use --no-dedupe-cap (or --dedupeCap 0) to disable. See `query dedupe-cap-events` for what fired.',
 		},
 		dedupeMapCap: {
 			type: 'number',
 			group: 'Scope & filtering',
-			desc: 'Hard cap on the number of distinct URL shapes --dedupe-cap tracks at once; the least-recently-touched shape is evicted beyond this. Only relevant when --dedupe-cap is set.',
+			desc: 'Hard cap on the number of distinct URL shapes --dedupe-cap tracks at once; the least-recently-touched shape is evicted beyond this. Only relevant when --dedupe-cap is enabled.',
 		},
 		interval: {
 			type: 'number',

packages/@nitpicker/cli/src/commands/pipeline.spec.ts

@@ -134,6 +134,11 @@ describe('pipeline command', () => {
 		);
 	});
 
+	it('dedupeCap フラグは commandDef 側も default: 10 で crawl.ts と揃えている（手書き複製ゆえの同期漏れガード）', async () => {
+		const { commandDef } = await import('./pipeline.js');
+		expect(commandDef.flags.dedupeCap.default).toBe(10);
+	});
+
 	it("forwards --dedupe-cap/--dedupe-map-cap to startCrawl (pipeline.ts hand-writes its own flags object rather than reusing crawl.ts's mapper, see the TODO on commandDef.flags)", async () => {
 		vi.mocked(startCrawlFn).mockResolvedValue('/tmp/site.nitpicker');
 		vi.mocked(analyzeFn).mockResolvedValue();

packages/@nitpicker/cli/src/commands/pipeline.ts

@@ -148,13 +148,14 @@ export const commandDef = {
 		},
 		dedupeCap: {
 			type: 'number',
+			default: 10,
 			group: 'Crawl options',
-			desc: 'Same-cluster soft cap: stop enqueueing newly-discovered internal URLs whose shape (e.g. `/news/date/{n}/`) has accumulated this many matching-title/description/og-tag observations. Opt-in — omit to disable. Backstop against a site that keeps serving 2xx for a self-generating pager/query-parameter trap; see `query dedupe-cap-events` for what fired.',
+			desc: 'Same-cluster soft cap: stop enqueueing newly-discovered internal URLs whose shape (e.g. `/news/date/{n}/`) has accumulated this many matching-title/description/og-tag observations. On by default (10) as a backstop against a site that keeps serving 2xx for a self-generating pager/query-parameter trap — false positives on legitimate large sections are structurally prevented (each such page differs in title/og tags, so the majority-vote counter never accumulates). Use --no-dedupe-cap (or --dedupeCap 0) to disable. See `query dedupe-cap-events` for what fired.',
 		},
 		dedupeMapCap: {
 			type: 'number',
 			group: 'Crawl options',
-			desc: 'Hard cap on the number of distinct URL shapes --dedupe-cap tracks at once; the least-recently-touched shape is evicted beyond this. Only relevant when --dedupe-cap is set.',
+			desc: 'Hard cap on the number of distinct URL shapes --dedupe-cap tracks at once; the least-recently-touched shape is evicted beyond this. Only relevant when --dedupe-cap is enabled.',
 		},
 		// analyze flags
 		all: {

packages/@nitpicker/cli/src/commands/query.ts

@@ -81,6 +81,10 @@ export const commandDef = {
 			type: 'boolean',
 			desc: 'Filter to pages with noindex',
 		},
+		isDedupeCapped: {
+			type: 'boolean',
+			desc: 'Filter to pages whose URL shape --dedupe-cap captured as a same-cluster crawl trap',
+		},
 		urlPattern: {
 			type: 'string',
 			valueName: 'pattern',
@@ -205,6 +209,7 @@ export const commandDef = {
 				'missingTitle',
 				'missingDescription',
 				'noindex',
+				'isDedupeCapped',
 				'urlPattern',
 				'directory',
 				'sortBy',

packages/@nitpicker/cli/src/commands/viewer-build.spec.ts

@@ -30,12 +30,14 @@ const mockBuildViewerReadModel = vi.fn().mockResolvedValue();
 const mockEnsureViewerReadModel = vi.fn().mockResolvedValue();
 const mockBackfillBodyHashFromHtmlBlobs = vi.fn().mockResolvedValue();
 const mockBackfillAliasOfId = vi.fn().mockResolvedValue();
+const mockBackfillDedupeCapEventId = vi.fn().mockResolvedValue();
 
 vi.mock('@nitpicker/query', () => ({
 	buildViewerReadModel: mockBuildViewerReadModel,
 	ensureViewerReadModel: mockEnsureViewerReadModel,
 	backfillBodyHashFromHtmlBlobs: mockBackfillBodyHashFromHtmlBlobs,
 	backfillAliasOfId: mockBackfillAliasOfId,
+	backfillDedupeCapEventId: mockBackfillDedupeCapEventId,
 }));
 
 const mockFormatCliError = vi.fn();
@@ -272,6 +274,57 @@ describe('viewerBuild command', () => {
 		);
 	});
 
+	it('always calls backfillDedupeCapEventId regardless of the --force branch', async () => {
+		// Regression guard, same reasoning as backfillAliasOfId's: an
+		// already-current read model skips buildViewerReadModel entirely, but
+		// a later --append/--retry-failed re-crawl can still add new
+		// dedupe_cap_events rows or newly-discovered pages matching an
+		// existing shape, which only this unconditional call catches up.
+		const { viewerBuild } = await import('./viewer-build.js');
+		await viewerBuild(['/tmp/existing.nitpicker'], {} as never);
+		expect(mockBackfillDedupeCapEventId).toHaveBeenCalledOnce();
+
+		vi.clearAllMocks();
+		mockCopyFile.mockResolvedValue();
+		mockUnlink.mockResolvedValue();
+		mockArchiveOpen.mockResolvedValue({
+			write: mockArchiveWrite,
+			close: mockArchiveClose,
+		});
+		mockArchiveWrite.mockResolvedValue();
+		mockArchiveClose.mockResolvedValue();
+		mockBackfillDedupeCapEventId.mockResolvedValue();
+		mockExistsSync.mockImplementation((p: string) => !p.endsWith('.bak'));
+		mockStatSync.mockReturnValue({ isFile: () => true });
+		const { viewerBuild: viewerBuildAgain } = await import('./viewer-build.js');
+		await viewerBuildAgain(['/tmp/existing.nitpicker'], { force: true } as never);
+		expect(mockBackfillDedupeCapEventId).toHaveBeenCalledOnce();
+	});
+
+	it('runs backfillDedupeCapEventId after backfillAliasOfId and before archive.write()', async () => {
+		const { viewerBuild } = await import('./viewer-build.js');
+		await viewerBuild(['/tmp/existing.nitpicker'], {} as never);
+
+		const aliasOrder = mockBackfillAliasOfId.mock.invocationCallOrder[0];
+		const dedupeCapOrder = mockBackfillDedupeCapEventId.mock.invocationCallOrder[0];
+		const writeOrder = mockArchiveWrite.mock.invocationCallOrder[0];
+		expect(aliasOrder!).toBeLessThan(dedupeCapOrder!);
+		expect(dedupeCapOrder!).toBeLessThan(writeOrder!);
+	});
+
+	it('logs backfillDedupeCapEventId progress to stderr', async () => {
+		mockBackfillDedupeCapEventId.mockImplementation((_archive, onProgress) => {
+			onProgress(1, 3);
+			return Promise.resolve();
+		});
+		const { viewerBuild } = await import('./viewer-build.js');
+		await viewerBuild(['/tmp/existing.nitpicker'], {} as never);
+
+		expect(consoleErrorSpy).toHaveBeenCalledWith(
+			'[nitpicker] content_items.dedupe_cap_event_id backfill: 1/3',
+		);
+	});
+
 	it('takes a backup before opening the archive writably', async () => {
 		const { viewerBuild } = await import('./viewer-build.js');
 		await viewerBuild(['/tmp/existing.nitpicker'], {} as never);

packages/@nitpicker/cli/src/commands/viewer-build.ts

@@ -8,6 +8,7 @@ import { Archive } from '@nitpicker/crawler';
 import {
 	backfillAliasOfId,
 	backfillBodyHashFromHtmlBlobs,
+	backfillDedupeCapEventId,
 	buildViewerReadModel,
 	ensureViewerReadModel,
 } from '@nitpicker/query';
@@ -169,6 +170,22 @@ export async function viewerBuild(
 					`[nitpicker] content_items.alias_of_id backfill: ${processed}/${total}`,
 				);
 			});
+			// Unlike the two backfills above, `dedupe_cap_event_id`'s initial
+			// rollout IS covered by a read-model schema bump (`viewer_pages.
+			// is_dedupe_capped` needs one) — but the same gate-bypass problem
+			// resurfaces on every later `--append`/`--retry-failed` re-crawl
+			// of an already-current archive: new `dedupe_cap_events` rows or
+			// newly-discovered pages matching an existing shape would never
+			// get (re-)marked, since `ensureViewerReadModel`'s version check
+			// only answers "did the schema change," not "did the underlying
+			// data." Called unconditionally here for that ongoing-maintenance
+			// case, same as `backfillBodyHashFromHtmlBlobs`/`backfillAliasOfId`.
+			await backfillDedupeCapEventId(archive, (processed, total) => {
+				// eslint-disable-next-line no-console
+				console.error(
+					`[nitpicker] content_items.dedupe_cap_event_id backfill: ${processed}/${total}`,
+				);
+			});
 			await archive.write();
 		} finally {
 			await archive.close();

packages/@nitpicker/cli/src/crawl/map-flags-to-crawl-config.spec.ts

@@ -90,10 +90,24 @@ describe('mapFlagsToCrawlConfig', () => {
 	});
 
 	it('dedupeCap 未指定は null（無効化）にマッピングする', () => {
+		// Real CLI usage never reaches this branch — roar/yargs-parser applies
+		// the flag's own `default: 10` before this function ever sees `flags`
+		// — but the function stays defensive for callers that construct
+		// `flags` directly (e.g. this test suite, or a future non-CLI caller).
 		const result = mapFlagsToCrawlConfig({});
 		expect(result.dedupeCap).toBeNull();
 	});
 
+	it('dedupeCap: 0 は null（無効化）にマッピングする — --no-dedupe-cap の実際の値', () => {
+		// yargs-parser's boolean-negation coercion turns `--no-dedupe-cap`
+		// into `dedupeCap: 0` (Number(false)), not `undefined`. Without this
+		// conversion, `0` would flow through to `DedupeCapTracker`, whose
+		// `computeEffectiveThreshold` floors any positive threshold at 1 —
+		// capping on the very first observation, the opposite of disabling.
+		const result = mapFlagsToCrawlConfig({ dedupeCap: 0 });
+		expect(result.dedupeCap).toBeNull();
+	});
+
 	it('dedupeMapCap を指定した値のままマッピングする', () => {
 		const result = mapFlagsToCrawlConfig({ dedupeMapCap: 50_000 });
 		expect(result.dedupeMapCap).toBe(50_000);

packages/@nitpicker/cli/src/crawl/map-flags-to-crawl-config.ts

@@ -29,7 +29,16 @@ export function mapFlagsToCrawlConfig(flags: CrawlFlagInput) {
 		excludes: flags.exclude,
 		excludeKeywords: flags.excludeKeyword,
 		excludeUrls: flags.excludeUrl,
-		dedupeCap: flags.dedupeCap ?? null,
+		// `0` means "disabled" here, not "cap after 0 observations" — the only
+		// way to reach `dedupeCap: 0` is `--no-dedupe-cap` (yargs-parser's
+		// boolean-negation numeric coercion) or an explicit `--dedupeCap 0`,
+		// both of which mean "turn this off." `DedupeCapTracker`'s own
+		// `computeEffectiveThreshold` floors any positive threshold at 1, so
+		// passing `0` through unchanged would cap on the very first
+		// observation — the opposite of disabling. `null` is the sentinel
+		// every downstream `!== null` gate (`crawler.ts`, `crawler-orchestrator.ts`)
+		// already treats as "tracker not constructed."
+		dedupeCap: flags.dedupeCap || null,
 		dedupeMapCap: flags.dedupeMapCap,
 	};
 }

packages/@nitpicker/cli/src/query/map-flags-to-query-options.spec.ts

@@ -28,6 +28,7 @@ describe('mapFlagsToQueryOptions', () => {
 			missingTitle: true,
 			missingDescription: undefined,
 			noindex: undefined,
+			isDedupeCapped: undefined,
 			urlPattern: undefined,
 			directory: undefined,
 			sortBy: 'status',
@@ -477,6 +478,7 @@ describe('mapFlagsToQueryOptions', () => {
 			missingTitle: true,
 			missingDescription: true,
 			noindex: true,
+			isDedupeCapped: true,
 			missingAlt: true,
 			missingDimensions: true,
 			validator: 'axe',

packages/@nitpicker/cli/src/query/map-flags-to-query-options.ts

@@ -69,6 +69,7 @@ export function mapFlagsToQueryOptions(
 				missingTitle: flags.missingTitle,
 				missingDescription: flags.missingDescription,
 				noindex: flags.noindex,
+				isDedupeCapped: flags.isDedupeCapped,
 				urlPattern: flags.urlPattern,
 				directory: flags.directory,
 				sortBy: flags.sortBy as 'url' | 'status' | 'title' | undefined,

packages/@nitpicker/crawler/src/archive/create-entity-tables.spec.ts

@@ -1,13 +1,20 @@
 import knex from 'knex';
 import { describe, it, expect } from 'vitest';
 
+import { createAdjunctTables } from './create-adjunct-tables.js';
 import { createEntityTables } from './create-entity-tables.js';
 import { createRefTables } from './create-ref-tables.js';
 import { LibsqlDialect } from './libsql-dialect.js';
 
 /**
  * Creates the 0.13 ref tables that the entity tables reference, then runs
- * the entity-table DDL against a fresh in-memory database.
+ * the entity-table DDL against a fresh in-memory database — plus the
+ * adjunct tables, since `content_items.dedupe_cap_event_id REFERENCES
+ * dedupe_cap_events(id)` needs that table to exist before any `content_items`
+ * write is even preparable (SQLite must resolve a declared FK's target
+ * table to compile the statement, independent of whether the enforcement
+ * pragma is on). Mirrors `initSchema`, which always creates entity and
+ * adjunct tables together.
  * @param options - `foreignKeys: true` enables `PRAGMA foreign_keys = ON`
  *   before the caller runs INSERTs (required for any test that exercises
  *   FK / CASCADE / DEFERRABLE / CHECK behaviour).
@@ -27,6 +34,7 @@ async function openDbWithEntityTables(
 	}
 	await createRefTables(db);
 	await createEntityTables(db);
+	await createAdjunctTables(db);
 	return db;
 }
 
@@ -103,6 +111,7 @@ describe('createEntityTables', () => {
 			'header_set_id',
 			'redirect_dest_id',
 			'alias_of_id',
+			'dedupe_cap_event_id',
 			'source',
 			'first_crawled_at',
 			'last_crawled_at',

packages/@nitpicker/crawler/src/archive/create-entity-tables.ts

@@ -200,6 +200,7 @@ export async function createEntityTables(instance: Knex): Promise<void> {
 			header_set_id    INTEGER REFERENCES header_sets(id),
 			redirect_dest_id INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED,
 			alias_of_id      INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED,
+			dedupe_cap_event_id INTEGER REFERENCES dedupe_cap_events(id) DEFERRABLE INITIALLY DEFERRED,
 			source           TEXT NOT NULL DEFAULT 'crawled',
 			first_crawled_at INTEGER,
 			last_crawled_at  INTEGER,
@@ -226,6 +227,15 @@ export async function createEntityTables(instance: Knex): Promise<void> {
 	// `migrateContentItemsAliasOfId` instead, which runs after the
 	// column-add guard for both fresh and legacy archives (same reasoning as
 	// `page_meta.body_hash`'s index).
+	//
+	// `dedupe_cap_event_id` gets no index anywhere, not even in its own
+	// migration (`migrateContentItemsDedupeCapEventId`) — unlike
+	// `alias_of_id`, there is no known hot read path filtering on this
+	// column yet (`--dedupe-cap` is opt-in and the marked row count is
+	// small: capped shapes × matching URLs). Adding a speculative index
+	// without a measured query to justify it violates this archive's
+	// "no speculative index" rule; add one later with `EXPLAIN QUERY PLAN`
+	// evidence if a real hot path emerges.
 	await instance.raw(
 		'CREATE INDEX IF NOT EXISTS idx_content_items_content_type_id ON content_items(content_type_id)',
 	);

packages/@nitpicker/crawler/src/archive/db-ops/_shared/resolve-content-item-id.spec.ts

@@ -3,6 +3,7 @@ import type { Knex } from 'knex';
 import knex from 'knex';
 import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 
+import { createAdjunctTables } from '../../create-adjunct-tables.js';
 import { createEntityTables } from '../../create-entity-tables.js';
 import { createRefTables } from '../../create-ref-tables.js';
 import { LibsqlDialect } from '../../libsql-dialect.js';
@@ -21,6 +22,7 @@ describe('resolveContentItemId', () => {
 		});
 		await createRefTables(db);
 		await createEntityTables(db);
+		await createAdjunctTables(db);
 	});
 
 	afterEach(async () => {