diff --git a/lib/cron-heartbeat.ts b/lib/cron-heartbeat.ts index 6df0fa4..864684e 100644 --- a/lib/cron-heartbeat.ts +++ b/lib/cron-heartbeat.ts @@ -102,3 +102,22 @@ export function getAllHeartbeats(): Heartbeat[] { runCount: r.run_count, })); } + +/** + * Off-host state of the daily DB backup, carried inside its heartbeat note. + * + * The note itself is prose for whoever reads /health. This token is the one + * part `lib/health.ts` parses, and it lives here — beside the contract it + * rides on — so the writer and the reader cannot drift apart into two string + * literals in two files. + */ +export type OffHostState = "none" | "ok" | "fail"; + +export function offHostToken(state: OffHostState): string { + return `offhost=${state}`; +} + +export function readOffHost(note: string | null | undefined): OffHostState | null { + const m = /\boffhost=(none|ok|fail)\b/.exec(note ?? ""); + return m ? (m[1] as OffHostState) : null; +} diff --git a/lib/health.ts b/lib/health.ts index 00ac5dd..a40077f 100644 --- a/lib/health.ts +++ b/lib/health.ts @@ -10,7 +10,7 @@ import fs from "node:fs"; import path from "node:path"; import { getDb } from "./db"; import { rateLimitSnapshot } from "./rate-limit"; -import { getHeartbeat } from "./cron-heartbeat"; +import { getHeartbeat, readOffHost } from "./cron-heartbeat"; import { assetCoverage, isCallableAsset } from "./prices"; import { todayAiSpendUsd } from "./grok"; import { recentAuditRuns, type AuditRun } from "./audit-log"; @@ -132,6 +132,34 @@ function applyContentStaleness( }; } +/** + * Third opinion, for the backup only: did the copy leave the box? + * + * Same failure shape as the one above. The heartbeat answers "did the cron + * run" and the verification answers "would this file restore" — neither + * answers "does a copy exist anywhere the host's disk isn't". Snapshots land + * in ~/backups/alpha, on the same volume as the DB they copy, so with + * BACKUP_REMOTE unset the row reported `ok` for something one disk failure + * away from nothing. + * + * Capped at `warn`, never `fail`, for the same reason as content staleness: + * the cron is healthy and ?strict=1 must not 503 a monitor over a + * configuration gap. But `ok` overstated it, and this is the one row whose + * whole job is to be true before someone needs it. + */ +function applyOffHostGap( + subsystem: SubsystemHealth, + heartbeat: ReturnType +): SubsystemHealth { + if (subsystem.status === "fail") return subsystem; + if (readOffHost(heartbeat?.lastNote) !== "none") return subsystem; + return { + ...subsystem, + status: "warn", + note: `${subsystem.note ? subsystem.note + " " : ""}사본이 원본과 같은 호스트에 있습니다 — BACKUP_REMOTE 미설정. 호스트 손실 시 둘 다 사라집니다.`, + }; +} + /** Event-driven crons may be legitimately quiet for days; a fortnight of * nothing is a warning, six weeks is a stopped subsystem. */ const CONTENT_WARN_SEC = 14 * 24 * 3600; @@ -536,7 +564,7 @@ export function getSystemHealth(): { ? `마지막 실행 ${hb.lastStatus}. ${hb.lastNote ?? ""}`.trim() : "heartbeat 없음 — cron 첫 실행 대기 중.", }); - return applyHeartbeatFailure(sub, hb); + return applyOffHostGap(applyHeartbeatFailure(sub, hb), hb); })(), (() => { // Liveness of the weekly audit cron — deliberately NOT its citation diff --git a/scripts/backup-db.ts b/scripts/backup-db.ts index 945d76e..b618f01 100644 --- a/scripts/backup-db.ts +++ b/scripts/backup-db.ts @@ -41,6 +41,9 @@ import path from "node:path"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { loadScriptEnv } from "../lib/script-env"; +// Type-only: the value side stays behind the dynamic import below, which +// must not run before loadScriptEnv() has put DB_PATH in the environment. +import type { OffHostState } from "../lib/cron-heartbeat"; loadScriptEnv(); @@ -124,7 +127,7 @@ async function main() { return; } - const { recordHeartbeat } = await import("../lib/cron-heartbeat"); + const { recordHeartbeat, offHostToken } = await import("../lib/cron-heartbeat"); const src = process.env.DB_PATH; if (!src || !fs.existsSync(src)) { const note = `DB_PATH 없음 또는 파일 없음: ${src ?? "(unset)"}`; @@ -180,6 +183,7 @@ async function main() { // warning — the local copy still exists, but the box is a single point of // failure again and someone has to know. let offHost = "off-host 미설정 (BACKUP_REMOTE)"; + let offHostState: OffHostState = "none"; let status: "ok" | "error" = "ok"; if (remote) { const rsync = process.env.BACKUP_RSYNC_BIN || "rsync"; @@ -188,9 +192,11 @@ async function main() { timeout: 15 * 60_000, }); offHost = `off-host 복사 완료 → ${remote}`; + offHostState = "ok"; console.log(offHost); } catch (err) { offHost = `off-host 복사 실패 → ${remote}: ${(err as Error).message.slice(0, 200)}`; + offHostState = "fail"; console.error(offHost); status = "error"; } @@ -201,7 +207,7 @@ async function main() { } const removed = prune(dir, keep); - const note = `${path.basename(dest)} ${sizeMb}MB · ${check.note} · ${offHost} · 정리 ${removed}건 (보관 ${keep})`; + const note = `${offHostToken(offHostState)} · ${path.basename(dest)} ${sizeMb}MB · ${check.note} · ${offHost} · 정리 ${removed}건 (보관 ${keep})`; console.log(`Heartbeat: ${status} — ${note}`); recordHeartbeat("alpha-backup-cron", status, note); if (status === "error") process.exitCode = 1; diff --git a/tests/offhost.test.ts b/tests/offhost.test.ts new file mode 100644 index 0000000..1dbbb54 --- /dev/null +++ b/tests/offhost.test.ts @@ -0,0 +1,65 @@ +/** + * The off-host marker in the backup heartbeat note. + * + * scripts/backup-db.ts writes this token and lib/health.ts reads it, and the + * whole point of the row is to be true before someone needs it. A silent + * drift between writer and reader would put the row back where it was — + * reporting `ok` for a copy sitting on the same disk as the original. + * + * Pure string work. The DB_PATH dance below exists only because + * lib/cron-heartbeat.ts imports lib/db, which mkdirs its data directory at + * module load; the temp path keeps that out of the checkout. + */ + +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { after, before, describe, it } from "node:test"; + +type Heartbeat = typeof import("../lib/cron-heartbeat"); + +let hb: Heartbeat; +let tmpDir: string; + +before(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "alpha-offhost-test-")); + process.env.DB_PATH = path.join(tmpDir, "test.sqlite"); + hb = await import("../lib/cron-heartbeat"); +}); + +after(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +describe("off-host marker", () => { + it("round-trips every state", () => { + for (const state of ["none", "ok", "fail"] as const) { + assert.equal(hb.readOffHost(hb.offHostToken(state)), state); + } + }); + + it("reads the note backup-db.ts actually writes", () => { + // Verbatim shape from scripts/backup-db.ts: token first, then prose. + const note = + "offhost=none · alpha-daily-20260825T180000Z.sqlite 17.6MB · " + + "integrity ok, posts=978 · off-host 미설정 (BACKUP_REMOTE) · 정리 0건 (보관 14)"; + assert.equal(hb.readOffHost(note), "none"); + }); + + it("follows the token, not the prose", () => { + // The prose says 미설정 and the token says otherwise. The token wins, + // because prose is for people and gets rewritten; this is the contract. + const note = "offhost=ok · off-host 미설정 이라는 옛 문구가 남아 있어도"; + assert.equal(hb.readOffHost(note), "ok"); + }); + + it("returns null when there is no token", () => { + // Heartbeats written before the token existed carry no verdict, and an + // absent token is not evidence of an absent copy. health.ts leaves the + // row alone rather than guessing; the next 03:00 run supplies the truth. + assert.equal(hb.readOffHost("integrity ok, posts=978"), null); + assert.equal(hb.readOffHost(null), null); + assert.equal(hb.readOffHost(undefined), null); + }); +});