Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions lib/cron-heartbeat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -102,3 +102,22 @@ export function getAllHeartbeats(): Heartbeat[] {
runCount: r.run_count,
}));
}

/**
* Off-host state of the daily DB backup, carried inside its heartbeat note.
*
* The note itself is prose for whoever reads /health. This token is the one
* part `lib/health.ts` parses, and it lives here — beside the contract it
* rides on — so the writer and the reader cannot drift apart into two string
* literals in two files.
*/
export type OffHostState = "none" | "ok" | "fail";

export function offHostToken(state: OffHostState): string {
return `offhost=${state}`;
}

export function readOffHost(note: string | null | undefined): OffHostState | null {
const m = /\boffhost=(none|ok|fail)\b/.exec(note ?? "");
return m ? (m[1] as OffHostState) : null;
}
32 changes: 30 additions & 2 deletions lib/health.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import fs from "node:fs";
import path from "node:path";
import { getDb } from "./db";
import { rateLimitSnapshot } from "./rate-limit";
import { getHeartbeat } from "./cron-heartbeat";
import { getHeartbeat, readOffHost } from "./cron-heartbeat";
import { assetCoverage, isCallableAsset } from "./prices";
import { todayAiSpendUsd } from "./grok";
import { recentAuditRuns, type AuditRun } from "./audit-log";
Expand Down Expand Up @@ -132,6 +132,34 @@ function applyContentStaleness(
};
}

/**
* Third opinion, for the backup only: did the copy leave the box?
*
* Same failure shape as the one above. The heartbeat answers "did the cron
* run" and the verification answers "would this file restore" — neither
* answers "does a copy exist anywhere the host's disk isn't". Snapshots land
* in ~/backups/alpha, on the same volume as the DB they copy, so with
* BACKUP_REMOTE unset the row reported `ok` for something one disk failure
* away from nothing.
*
* Capped at `warn`, never `fail`, for the same reason as content staleness:
* the cron is healthy and ?strict=1 must not 503 a monitor over a
* configuration gap. But `ok` overstated it, and this is the one row whose
* whole job is to be true before someone needs it.
*/
function applyOffHostGap(
subsystem: SubsystemHealth,
heartbeat: ReturnType<typeof getHeartbeat>
): SubsystemHealth {
if (subsystem.status === "fail") return subsystem;
if (readOffHost(heartbeat?.lastNote) !== "none") return subsystem;
return {
...subsystem,
status: "warn",
note: `${subsystem.note ? subsystem.note + " " : ""}사본이 원본과 같은 호스트에 있습니다 — BACKUP_REMOTE 미설정. 호스트 손실 시 둘 다 사라집니다.`,
};
}

/** Event-driven crons may be legitimately quiet for days; a fortnight of
* nothing is a warning, six weeks is a stopped subsystem. */
const CONTENT_WARN_SEC = 14 * 24 * 3600;
Expand Down Expand Up @@ -536,7 +564,7 @@ export function getSystemHealth(): {
? `마지막 실행 ${hb.lastStatus}. ${hb.lastNote ?? ""}`.trim()
: "heartbeat 없음 — cron 첫 실행 대기 중.",
});
return applyHeartbeatFailure(sub, hb);
return applyOffHostGap(applyHeartbeatFailure(sub, hb), hb);
})(),
(() => {
// Liveness of the weekly audit cron — deliberately NOT its citation
Expand Down
10 changes: 8 additions & 2 deletions scripts/backup-db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ import path from "node:path";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { loadScriptEnv } from "../lib/script-env";
// Type-only: the value side stays behind the dynamic import below, which
// must not run before loadScriptEnv() has put DB_PATH in the environment.
import type { OffHostState } from "../lib/cron-heartbeat";

loadScriptEnv();

Expand Down Expand Up @@ -124,7 +127,7 @@ async function main() {
return;
}

const { recordHeartbeat } = await import("../lib/cron-heartbeat");
const { recordHeartbeat, offHostToken } = await import("../lib/cron-heartbeat");
const src = process.env.DB_PATH;
if (!src || !fs.existsSync(src)) {
const note = `DB_PATH 없음 또는 파일 없음: ${src ?? "(unset)"}`;
Expand Down Expand Up @@ -180,6 +183,7 @@ async function main() {
// warning — the local copy still exists, but the box is a single point of
// failure again and someone has to know.
let offHost = "off-host 미설정 (BACKUP_REMOTE)";
let offHostState: OffHostState = "none";
let status: "ok" | "error" = "ok";
if (remote) {
const rsync = process.env.BACKUP_RSYNC_BIN || "rsync";
Expand All @@ -188,9 +192,11 @@ async function main() {
timeout: 15 * 60_000,
});
offHost = `off-host 복사 완료 → ${remote}`;
offHostState = "ok";
console.log(offHost);
} catch (err) {
offHost = `off-host 복사 실패 → ${remote}: ${(err as Error).message.slice(0, 200)}`;
offHostState = "fail";
console.error(offHost);
status = "error";
}
Expand All @@ -201,7 +207,7 @@ async function main() {
}

const removed = prune(dir, keep);
const note = `${path.basename(dest)} ${sizeMb}MB · ${check.note} · ${offHost} · 정리 ${removed}건 (보관 ${keep})`;
const note = `${offHostToken(offHostState)} · ${path.basename(dest)} ${sizeMb}MB · ${check.note} · ${offHost} · 정리 ${removed}건 (보관 ${keep})`;
console.log(`Heartbeat: ${status} — ${note}`);
recordHeartbeat("alpha-backup-cron", status, note);
if (status === "error") process.exitCode = 1;
Expand Down
65 changes: 65 additions & 0 deletions tests/offhost.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/**
* The off-host marker in the backup heartbeat note.
*
* scripts/backup-db.ts writes this token and lib/health.ts reads it, and the
* whole point of the row is to be true before someone needs it. A silent
* drift between writer and reader would put the row back where it was —
* reporting `ok` for a copy sitting on the same disk as the original.
*
* Pure string work. The DB_PATH dance below exists only because
* lib/cron-heartbeat.ts imports lib/db, which mkdirs its data directory at
* module load; the temp path keeps that out of the checkout.
*/

import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { after, before, describe, it } from "node:test";

type Heartbeat = typeof import("../lib/cron-heartbeat");

let hb: Heartbeat;
let tmpDir: string;

before(async () => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "alpha-offhost-test-"));
process.env.DB_PATH = path.join(tmpDir, "test.sqlite");
hb = await import("../lib/cron-heartbeat");
});

after(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
});

describe("off-host marker", () => {
it("round-trips every state", () => {
for (const state of ["none", "ok", "fail"] as const) {
assert.equal(hb.readOffHost(hb.offHostToken(state)), state);
}
});

it("reads the note backup-db.ts actually writes", () => {
// Verbatim shape from scripts/backup-db.ts: token first, then prose.
const note =
"offhost=none · alpha-daily-20260825T180000Z.sqlite 17.6MB · " +
"integrity ok, posts=978 · off-host 미설정 (BACKUP_REMOTE) · 정리 0건 (보관 14)";
assert.equal(hb.readOffHost(note), "none");
});

it("follows the token, not the prose", () => {
// The prose says 미설정 and the token says otherwise. The token wins,
// because prose is for people and gets rewritten; this is the contract.
const note = "offhost=ok · off-host 미설정 이라는 옛 문구가 남아 있어도";
assert.equal(hb.readOffHost(note), "ok");
});

it("returns null when there is no token", () => {
// Heartbeats written before the token existed carry no verdict, and an
// absent token is not evidence of an absent copy. health.ts leaves the
// row alone rather than guessing; the next 03:00 run supplies the truth.
assert.equal(hb.readOffHost("integrity ok, posts=978"), null);
assert.equal(hb.readOffHost(null), null);
assert.equal(hb.readOffHost(undefined), null);
});
});
Loading