diff --git a/.agents/skills/afk/SKILL.md b/.agents/skills/afk/SKILL.md index 95f64b11e0..4e33324fab 100644 --- a/.agents/skills/afk/SKILL.md +++ b/.agents/skills/afk/SKILL.md @@ -61,6 +61,7 @@ No `/back` is needed. The first genuine message is the return signal: - A message **without** the current operational prefix or a legacy bare marker, and **not** starting with `/afk` -> the captain is back. Run `bin/fm-afk-return.sh` before acting on the message that brought the captain back. That script owns correct-ordered daemon shutdown, durable wake draining, escalation and wedge evidence, and the return-catch-up gate. + Its `away-telegram` evidence records carry the escalation lines an accepted Telegram notice already delivered to the captain's phone, so they never appeared in chat; act on them like any other catch-up evidence rather than re-alerting him. If it reports a firstmate-actionable `blocked:` event, remediate it immediately through the normal lifecycle, or explicitly reclassify it with a durable reason and close its decision key with `resolved [key=...]`, then run `bin/fm-afk-return.sh check`. Once the daemon stops, resume full per-wake responsiveness through the emitted primary-harness supervision protocol while blocker handling proceeds, so the gate never creates a blind wait. Do not answer a Bearings request or perform any other ordinary captain work until the check exits successfully. @@ -170,17 +171,20 @@ Classify each wake this way: captain-relevant status line the per-wake classifier might miss. - Unknown reason, or any uncertainty -> escalate fail-safe. -Escalations are buffered up to `FM_ESCALATE_BATCH_SECS` (default 90s; 0 = -immediate) and flushed as one single-line digest prefixed with the current -operational prefix, carrying pre-read status summaries and a recommended action. -The single-line format makes the submission unambiguous across harnesses, and -the operational prefix lets firstmate distinguish it from a real captain message. +Escalations are buffered up to `FM_ESCALATE_BATCH_SECS` (default 90s; 0 = immediate) and flushed as one batch. +When Telegram mode is configured, that same batch first goes through the existing phone-inbox `tg` client as a concise notice with an explicit reminder that the notice grants no approval. +`bin/fm-away-ledger-lib.sh` is the single owner of the batch ledger (`state/.subsuper-escalations.since`: batch identity plus `reserved`, `confirmed`, and `accounted` counts, an attempt ordinal, and a `retry-after` epoch - the schedule the `deferred` outcome below reads); the phone only ever gets lines past `reserved`, so a batch that grows while the receipt is wedged sends one notice per new increment and never repeats a delivered line. +An accepted and fully recorded send leaves only a non-secret internal delivery receipt in Firstmate chat, naming the batch's private away digests `state/tg-away-digest/-*.items`; read those to reconcile and act, and do not repeat the alert to the captain there. +Every other outcome preserves the not-yet-recorded items in the in-session escalation as the safe fallback and labels the outcome honestly: `uncertain` means the send may or may not have reached the captain (a watchdog timeout, or a client that ran and exited non-zero, which cannot be told apart from a lost response to an accepted send) so it keeps its reservation and the rest of that batch goes to chat only, `failed`/`unavailable` means nothing left the box, so he was definitively not contacted there and that attempt is retired for a scheduled retry of the same lines, `deferred` means such a retry is armed but its schedule has not elapsed yet, an accepted send with an incomplete digest carries its items in session once, and an unreadable ledger means nothing was sent to the phone. +Accepted, failed, and unavailable outcomes persist under `state/tg-away-delivery/` by opaque delivery id without message text, chat id, token, or raw sender diagnostics; the away digests are separate private 0600 working records that DO hold the distilled lines, are folded into the return catch-up evidence before retirement, and retire with the away session. +The ledger reserves lines before network I/O and advances `confirmed` only from durable accepted evidence, and only over a contiguous prefix, so an accepted increment never implies acceptance of an earlier uncertain line; a restart resolves an interrupted send from that evidence instead of guessing: accepted evidence confirms with no second alert, an unresolved attempt is retired as uncertain and never retried, and a reservation with no evidence is released so those lines can still reach the phone. +Without Telegram mode configured, away mode keeps its existing in-session fallback behavior. +The single-line internal format makes submission unambiguous across harnesses, and the operational prefix lets firstmate distinguish it from a real captain message. ## Injection hardening -- **Single-line digest** - embedded newlines are collapsed to a literal - separator before injection, so submission is unambiguous regardless of - harness. +- **Single-line digest** - embedded newlines are collapsed to a literal separator before phone delivery or injection, so submission is unambiguous regardless of harness. +- **No duplicate captain notice** - accepted Telegram delivery injects only an opaque receipt; firstmate reads the durable fleet records to act and does not repeat the alert in captain chat while away mode remains active. - **Composer guard on the supervisor pane** - before injecting, the daemon checks `pane_is_busy` (harness busy footer means agent mid-turn) and reads `fm_backend_composer_state` directly. Only `empty` permits injection; `pending` protects half-typed or swallowed input, and `unknown` protects unreadable panes and bare dead-shell prompts. Every other result preserves the buffer for retry, so the daemon never merges its digest into the captain's half-typed line or types it into a shell. @@ -232,6 +236,7 @@ the operational prefix lets firstmate distinguish it from a real captain message Treat `state/.subsuper-escalations`, its `.since` sidecar, and `state/.subsuper-inject-wedged` as session-scoped delivery artifacts, not as the durable work record. Always enter through `bin/fm-afk-launch.sh`, which clears prior-session artifacts only for a fresh entry and preserves the current session's buffer on refresh. Always exit through `bin/fm-afk-launch.sh stop`, which keeps `state/.afk` present through the daemon's shutdown flush and clears it last. +Never delete those artifacts, the away digests, or the batch versions by hand: entry and return retire them through `bin/fm-away-ledger-lib.sh`, and a retirement that fails deliberately holds the return-catch-up gate open. `docs/herdr-backend.md` "Away-mode supervisor support" owns the current mechanism, and `docs/verification/runtime-backends.md` "Away-mode transport" owns active evidence. ## Reliability properties diff --git a/.agents/skills/bearings/SKILL.md b/.agents/skills/bearings/SKILL.md index b2804c7290..42990edd04 100644 --- a/.agents/skills/bearings/SKILL.md +++ b/.agents/skills/bearings/SKILL.md @@ -1,6 +1,9 @@ --- name: bearings -description: Generate a "pick up where I left off" status report from firstmate's live fleet state. Use when the captain invokes /bearings or asks for a bearings report, morning brief, status report, catch-up, "where did I leave off", or "what's in the works". Reads bounded local fleet state cheaply, optionally checks open PRs when requested, composes a scannable dated report to data/status-report-.md, and surfaces a concise version in chat; it is read-mostly and must not tear down, merge, or mutate task state as a side effect of producing the brief. +description: >- + Generate a "pick up where I left off" fleet digest from firstmate's live fleet state. + Use when the captain invokes /bearings or asks for a bearings report, morning brief, status report, catch-up, "where did I leave off", or "what's in the works". + Plain /bearings is chat-only by default, while /bearings file explicitly writes the dated data/status-report-.md artifact; live PR enrichment remains opt-in and composes with file mode. user-invocable: true metadata: internal: true @@ -8,42 +11,58 @@ metadata: # bearings -Generate a complete standalone snapshot from the fleet's current state, so the captain can resume in one read after a break, a night, or a context reset. -The deliverable is a dated markdown file plus a concise chat summary that each stand on the current snapshot rather than an earlier report. -This skill is read-mostly. -It reads fleet state and writes exactly one report file. -It never tears down a task, merges a PR, dispatches new work, or mutates any task state as a side effect of producing the brief - those belong to the captain's explicit word and the normal task lifecycle. +Generate a complete current snapshot from the fleet's current state, so the captain can resume in one read after a break, a night, or a context reset. +Plain `/bearings` returns only the concise four-section chat digest. +Only `/bearings file` writes the dated markdown report artifact and then returns the concise four-section chat digest linked to that report. +This skill is operationally read-only in both modes. +It never tears down a task, merges a PR, dispatches new work, steers a worker, answers a decision, cleans up work, mutates backlog or task state, or writes any file except the single dated report in explicit file mode. + +## Invocation modes + +- Plain `/bearings` gathers a fresh bounded snapshot and renders the four-section chat digest without creating, deleting, reading, or replacing `data/status-report-.md`. +- `/bearings file` gathers a fresh bounded snapshot, replaces today's `data/status-report-.md` from scratch, and renders the four-section chat digest with a link or path to that report. +- Treat `file` only as an explicit invocation option in the slash command. +- Do not treat natural-language requests such as "write a report", "save this", "persist it", or "make a file" as file mode unless the invocation explicitly includes the standalone `file` option. +- When the captain asks to include PRs, pass the snapshot command's live-PR opt-in. +- `/bearings include PRs` remains chat-only and makes the live-PR opt-in. +- `/bearings file include PRs` writes the dated report and makes the live-PR opt-in. ## What it does 1. **Gather live fleet state with one deterministic command.** - Run `bin/fm-bearings-snapshot.sh` and read its compact output. - It is the single bounded, deterministic source for this report and renders TOON by default. - Do not hand-probe the snapshot schema and do not make ad-hoc `gh-axi`/`gh` calls to assemble fleet facts; this command already assembles them. + Run `bin/fm-bearings-snapshot.sh` at invocation time and read its compact output. + It is the single bounded, deterministic fleet-state source for Bearings and renders TOON by default. + Do not create or consult a second fleet-state reader, parser contract, status-event-tail interpretation, visible-session recap, ad-hoc project probe, or ad-hoc `gh-axi`/`gh` query. The command's header and `--help` output own its exact fields, bounds, opt-ins, and output contract. - When the captain asks to include PRs, use the command's live-PR opt-in; otherwise keep the default local-only read. - If the command is unavailable, fall back to `bin/fm-fleet-snapshot.sh --json` and `bin/fm-crew-state.sh `; never infer current state from a raw `tail` of `state/.status`, which is append-only wake-event history whose last line goes stale. - For registered secondmates, use the snapshot's structured-home classification and provenance; a parent event or bounded terminal contradiction is fallback evidence, never authority over readable structured home state. - Structured captain-held decisions come from `decision-hold-lifecycle` and appear under `decisions_open`; do not scrape reports or visual-review artifacts to supplement them. - A queued item under `gates` only becomes "next work" when its blocker is gone and its time/date gate has arrived; until then it stays queued with the reason. - The `(main-inventory)` gate is an action-free integrity warning rather than queued work: render it under Charted Next with the related `omitted` disclosure, never invent an Underway row from backlog-only state, and never move it into Captain's Call. - -2. **Compose the detailed report file around the four-section spine, adding the richer detail the chat leaves out.** - The gather step is deterministic; your judgment is scoped to the last mile only - ranking the command's facts by what matters right now and writing the scannable prose. + Keep the default local-only read unless the captain asks to include PRs. + For registered secondmates, use the snapshot's structured-home classification and provenance. + A parent event or bounded terminal contradiction is fallback evidence, never authority over readable structured home state. + Structured captain-held decisions come from `decision-hold-lifecycle` and appear under `decisions_open`. + Do not scrape reports, visual-review artifacts, raw status-event tails, or visible conversation history to supplement current state. + A queued item under `gates` only becomes "next work" when its blocker is gone and its time/date gate has arrived. + Until then it stays queued with the reason. + The `(main-inventory)` gate is an action-free integrity warning rather than queued work. + Render it under Charted Next with the related `omitted` disclosure, never invent an Underway row from backlog-only state, and never move it into Captain's Call. + +2. **Compose the four-section chat digest from the fresh snapshot.** + The gather step is deterministic; your judgment is scoped to ranking the command's facts by what matters right now and writing scannable captain-facing prose. + The chat response uses the four complete sections in the chat-response contract below, in the same order, each always present. + Plain mode stops here and writes no report artifact. + +3. **In explicit file mode only, compose and replace the detailed report file.** + The report uses the same four complete sections as the chat, in the same order, and adds the detail the chat omits. Never read an earlier `data/status-report-*.md` to decide what to omit, include, describe as changed, or call current. - The report uses the same four complete sections as the chat (see the chat-response contract below), in the same order, each always present, and adds the detail the chat omits: + Write the full report to `data/status-report-.md` using today's date. + If today's file already exists, delete it first, then create a new file from scratch. + This is the only write allowed by the skill. + The detailed report includes: - **Title** - `# Bearings - ` (use "Morning status" only when the captain specifically asks for a morning brief), followed by two or three sentences framing where things stand. - **Captain's Call** - every open decision summarized with its options from the structured decision record, plus each PR ready to merge and each needed credential or login, every PR with the full `https://...` URL, never a bare `#number`. - **Recently Landed** - the bounded current recent-completions baseline from structured state across the main fleet and every registered secondmate home, rendered in full on every run. - - **Underway** - each live direct report making progress, with its current state, and the plans / main pickup pointers worth reopening (`data//report.md` files, `.lavish/*.html` boards). + - **Underway** - each live direct report making progress, with its current state, and the plans or main pickup pointers worth reopening (`data//report.md` files, `.lavish/*.html` boards). - **Charted Next** - queued or gated work, including any main-inventory integrity warning, with each item's blocker, date, or integrity reason. - -3. **Write the dated report file so it persists, then surface the mandatory four-section digest in chat.** - - Write the full report to `data/status-report-.md` using today's date. - This is the required artifact; it lives in gitignored `data/`. - If today's file already exists, delete it first, then create a new file from scratch. - - The chat response is the concise four-section digest defined by the contract below: materially shorter than the report file, complete as a current snapshot, internally consistent with the file, and linked to that file for the full picture. - - For a richer review surface, optionally offer a Lavish board with `lavish-axi` when the report has enough structure to deserve one, but the markdown file is the required artifact and the four-section chat digest is the required minimum. + After writing the file, return the concise four-section chat digest and include the report path or link without adding a fifth section. + For a richer review surface, optionally offer a Lavish board with `lavish-axi` when the report has enough structure to deserve one, but only after the required digest is ready. ## Chat-response contract @@ -62,22 +81,27 @@ Every `/bearings` chat response renders EXACTLY these four sections, in THIS ord Rules that keep the contract unambiguous: - Every section ALWAYS renders, even when empty, with its short empty-state sentence; never omit a section. -- Every report and chat digest is a complete current snapshot, never a delta against a prior report. +- Every chat digest and file-mode report is a complete current snapshot, never a delta against a prior report. - Recently Landed always renders the bounded current baseline, even when the same completions appeared in an earlier report. - The four buckets are mutually exclusive, so every item is forced into exactly one: needs-your-action is Captain's Call, done is Recently Landed, self-progressing is Underway, and not-yet-started work or an action-free fleet-integrity warning is Charted Next. - The strict boundary keeps action-free items OUT of Captain's Call: a working or validating task, a queued item blocked on another task or a date, landed work, a completed scout's report pointer, a declared `paused:` external wait, and a bare recorded PR with no merge-ready signal each belong to one of the other three sections, never Captain's Call. - A secondmate's own row appears Underway only for `active_child_work`; `externally_held` belongs in Charted Next, and `unknown` belongs there as an unavailable-state gate unless its reason requires the captain's action. - Do not suppress separately projected decisions, landed records, or gates from a `partial-structured` home merely because that secondmate's own row is `unknown`. -- The chat follows `AGENTS.md` section 9 and carries one scannable line per item, each PR as the full `https://...` URL; detailed decisions, plans, full gate reasons, and evidence live only in the report file, which the chat links to, so the chat stays materially shorter than that file. +- Include the required direct address to the captain inside one item or empty-state sentence. +- Every PR appears as the full `https://...` URL; a shorthand `#number` is fine only as a back-reference after the full URL has already appeared in the same digest. +- The chat follows `AGENTS.md` section 9 and carries one scannable line per item. +- Detailed decisions, plans, full gate reasons, and evidence belong in the file only when file mode is explicit, so plain chat stays concise and file-mode chat stays materially shorter than that file. +- In file mode, include the report path or link inside the four-section digest without adding another heading. ## Tone and content rules -- This report is a private, captain-facing internal artifact that lives in gitignored `data/`, so unlike normal captain chat it MAY reference task ids, PR URLs, and repo names - the captain works with these directly and needs them to resume; keep it organized and scannable, not a raw dump. -- Every PR reference is a full `https://...` URL, never a bare `#number`; a shorthand `#number` is fine only as a back-reference after the full URL has already appeared in the same report. +- The optional file-mode report is a private, captain-facing internal artifact that lives in gitignored `data/`, so unlike normal captain chat it MAY reference task ids, PR URLs, and repo names. +- The captain works with those directly and needs them to resume; keep the report organized and scannable, not a raw dump. +- Every PR reference is a full `https://...` URL, never a bare `#number`. - Never include PHI or secret values; the report is an operational artifact, but it is still subject to the same security and compliance rules that govern everything else in this fleet. ## Supervision discipline -This skill is read-mostly and changes no fleet state. -Do not tear down a task, merge a PR, dispatch queued work, or mutate any `state/` or `data/` file other than the single report file as a side effect of generating the brief. +This skill changes no fleet state. +Do not tear down a task, merge a PR, dispatch queued work, steer a worker, answer a queued decision, clean up work, or mutate any `state/` or `data/` file other than the single report file in explicit file mode. If the state you read suggests an action - a PR ready to merge, a queued item whose gate has arrived, or a needs-decision finding - name it in its section and leave the action to the normal lifecycle and configured authority rather than taking it from inside this skill. diff --git a/AGENTS.md b/AGENTS.md index 8c4558c53c..17003f2d9a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,6 +105,9 @@ state/ volatile runtime signals; gitignored tg-watch.check.sh generated Telegram-mode inbox poll shim; present only when opted in (section 15) tg-offered/ generated Telegram-mode durable offered markers, one per pending note id; pruned when the note leaves pending (section 15) tg-poll/ generated Telegram-mode diagnostic dedupe markers: error (inbox) and claim-error (offer claim) + tg-away-delivery/ away-mode Telegram escalation delivery evidence, one text-free .status per attempt; never holds alert text, chat id, token, or sender output + tg-away-digest/ away-mode private 0600 working copies of the escalation lines an accepted Telegram notice carried; read these when a delivery receipt points at them, folded into return catch-up, and retired with the away session (bin/fm-away-ledger-lib.sh) + tg-away-versions/ away-mode immutable batch versions, one complete v./ copy of the escalation buffer, ledger sidecar, wedge marker, and digests per ledger transition, plus the single active/active.applied pointer and the .owner.lock every transition holds; the live artifacts are that pointer's projection, and retained versions fold into return catch-up and retire whole once it is acknowledged (bin/fm-away-ledger-lib.sh) pending-replies/ parent-owned secondmate pending-reply records (correlation id, delivery vs reply, recovery, escalation); fm-pending-reply-lib.sh x-inbox/ generated X-mode pending mention payloads; fmx-respond drains it (section 14) x-context/ generated X-mode durable per-request reply context and one-wake offer markers, keyed by request_id; survives inbox cleanup and expires within seven days (section 14; bin/fm-x-lib.sh) diff --git a/README.md b/README.md index 08f4b0b1a5..2487096a9f 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Launching a supported harness inside it instantiates your first mate - and makes - **Optional secondmates** - opt in to persistent second mates that run from isolated firstmate homes with their own `FM_HOME`, state, projects, and session lock, supervising project clones or a project-less firstmate-repo domain, kept on the primary firstmate version by guarded local fast-forwards and checked for live agent processes at session start. - **Event-driven, zero-token supervision** - a bash watcher sleeps on the fleet and wakes the first mate only when something needs you; verified primary harnesses also get a turn-end backstop that blocks or follows up on a blind stop when work is under way and supervision is not live. - **Optional X mode** - opt in with one local `.env` token so firstmate can answer your public `@myfirstmate` mentions, act on normal reversible mention requests through the same lifecycle as chat requests, acknowledge spawned work, and post up to three public-safe completion follow-ups within seven days for genuine milestones and the final outcome without changing non-X behavior; dry-run preview records would-be replies and dismissals locally before go-live. -- **Optional Telegram mode** - opt in with one local flag file so the messages you send your own phone-inbox Telegram bot reach firstmate as work or questions and get a real reply in the same chat, with up to three bounded completion follow-ups; it stays fully inert until you opt in, authorizes only reversible lifecycle actions from the phone, and keeps the bot token entirely in the separate phone-inbox deployment. +- **Optional Telegram mode** - opt in with one local flag file so the messages you send your own phone-inbox Telegram bot reach firstmate as work or questions and get a real reply in the same chat, with up to three bounded completion follow-ups; while you are away, each away-mode escalation batch also reaches that same chat once, as notice only that grants no approval; it stays fully inert until you opt in, authorizes only reversible lifecycle actions from the phone, and keeps the bot token entirely in the separate phone-inbox deployment. - **Guarded by construction** - the first mate is read-only over your projects except for the guarded paths authorized by [hard rule 1](AGENTS.md#1-identity-and-prime-directives), with fleet sync's safe branch pruning remaining part of the fleet-sync exception; crewmates make every project change behind the configured merge authority. - **Restart-proof** - all state lives on disk and in the active session backend (tmux by hard default, herdr or cmux when selected or auto-detected, zellij/orca when explicitly selected); kill the session anytime and the next one reconciles, including confirmed-dead secondmate agents, and carries on. @@ -172,10 +172,17 @@ Claude and grok use the slash form shown here; codex uses the same names with `$ | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------- | | `/afk` | Enter away-mode supervision: the sub-supervisor self-handles routine notifications in bash, escalates captain-relevant events and bounded declared-external-wait rechecks as batched digests, and actively alerts if delivery gets stuck while you step away | | `/ahoy` | Recap visible session events since the prior real captain message plus visibly unanswered captain decisions, falling back to Bearings when invoked as the session's first real captain message | -| `/bearings` | Generate a standalone current-status report from bounded local fleet and registered-secondmate state, with live PR enrichment only when requested, written to a dated file in `data/` and surfaced concisely in chat; read-mostly, mutates no task state | +| `/bearings` | Generate a concise four-section chat digest from bounded local fleet and registered-secondmate state; use `/bearings file` to also replace today's dated report in `data/`, and add `include PRs` when live PR enrichment is wanted | | `/updatefirstmate` | Self-update the running firstmate and its secondmates to the latest from origin with fast-forward-only pulls, then re-read instructions and nudge secondmates | | `/stow` | Sweep the session for uncaptured durable knowledge, route each finding to its disk home per AGENTS.md, file undone next steps to the backlog, and report what is now safe to reset | +Bearings invocation examples: + +- `/bearings` returns the fresh four-section digest in chat only. +- `/bearings include PRs` keeps chat-only mode and opts into live PR enrichment. +- `/bearings file` replaces today's `data/status-report-.md` from scratch and links it from the four-section chat digest. +- `/bearings file include PRs` combines the dated report with live PR enrichment. + Agent-only reference skills live under `.agents/skills/` and are loaded by firstmate at the trigger points named in [`AGENTS.md`](AGENTS.md). ### Two-tier skill layout diff --git a/bin/fm-afk-launch.sh b/bin/fm-afk-launch.sh index 3bc1a1cbac..05ac5a3614 100755 --- a/bin/fm-afk-launch.sh +++ b/bin/fm-afk-launch.sh @@ -89,6 +89,28 @@ set +e fm_afk_launch_log() { printf 'fm-afk-launch: %s\n' "$*" >&2; } +# Sweep the away batch version store of everything that was never published +# (abandoned staging builds, pointer temps, incomplete version directories). +# Safe at any moment, including while a daemon is live: it never touches a +# complete version or the active pointer. +fm_afk_launch_versions_sweep() { + away_ledger_version_gc "$FM_AFK_LAUNCH_STATE" \ + || fm_afk_launch_log "could not sweep incomplete away batch versions" + return 0 +} + +# The entry-boundary transaction, owned by the ledger: sweep, replay an +# interrupted active-pointer switch, and capture the predecessor unit as one +# immutable version whose name is printed. This is the only point in the launcher +# where a version's content may reach the live paths, and the owner itself refuses +# the switch (rc 3) if a daemon is live, so no post-start switch is possible even +# by mistake. The stale-artifact clear below runs only once this has succeeded. +fm_afk_launch_entry_capture() { + away_ledger_entry_capture "$FM_AFK_LAUNCH_STATE" \ + "$FM_AFK_LAUNCH_STATE/.subsuper-escalations" \ + "$FM_AFK_LAUNCH_STATE/.subsuper-inject-wedged" +} + fm_afk_launch_lock_owned() { local pid expected actual [ -d "$FM_AFK_LAUNCH_LOCK" ] || return 1 @@ -357,24 +379,30 @@ fm_afk_launch_reconcile() { fi } -fm_afk_launch_restore_backup() { # - local backup=$1 had_afk=$2 artifact result=0 - rm -f "$FM_AFK_LAUNCH_STATE/.afk" \ +# Roll the away lifecycle back to the immutable version published before this +# transaction began: ONE atomic active-pointer switch onto that whole version, +# never a per-file copy into the live paths. The away-mode flag is the +# launcher's own artifact, not part of the ledger's unit, so it is restored here +# from the value captured before the attempt. +# +# A refused switch (rc 3: a daemon is live) or a failed materialisation leaves +# the version published and retained, so its escalation text reaches the return +# catch-up instead of being lost - the transaction is reported as failed either +# way. +fm_afk_launch_rollback() { # + local version=$1 had_afk=$2 afk=$3 result=0 rc=0 + rm -f "$FM_AFK_LAUNCH_STATE/.afk" || result=1 + away_ledger_version_activate "$FM_AFK_LAUNCH_STATE" \ "$FM_AFK_LAUNCH_STATE/.subsuper-escalations" \ - "$FM_AFK_LAUNCH_STATE/.subsuper-escalations.since" \ - "$FM_AFK_LAUNCH_STATE/.subsuper-inject-wedged" || result=1 + "$FM_AFK_LAUNCH_STATE/.subsuper-inject-wedged" "$version" || rc=$? if [ "$had_afk" -eq 1 ]; then - cp "$backup/.afk" "$FM_AFK_LAUNCH_STATE/.afk" || result=1 + printf '%s\n' "$afk" > "$FM_AFK_LAUNCH_STATE/.afk" || result=1 fi - for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged; do - if [ -e "$backup/$artifact" ]; then - cp -p "$backup/$artifact" "$FM_AFK_LAUNCH_STATE/$artifact" || result=1 - fi - done - if [ "$result" -eq 0 ]; then - rm -rf "$backup" || return 1 + if [ "$rc" -eq 0 ]; then + away_ledger_version_retire "$FM_AFK_LAUNCH_STATE" "$version" || result=1 else - fm_afk_launch_log "rollback restoration incomplete; backup retained at $backup" + result=1 + fm_afk_launch_log "rollback could not switch to away batch version $version; it is retained for the return catch-up" fi return "$result" } @@ -461,7 +489,7 @@ fm_afk_launch_create_tmux() { # } fm_afk_launch_start() { - local captain_target captain_backend backup artifact had_afk=0 result + local captain_target captain_backend version afk_content='' had_afk=0 result if [ -e "$FM_AFK_LAUNCH_STATE/.afk-return-catchup" ]; then fm_afk_launch_log "return catch-up is still pending; run bin/fm-afk-return.sh check before re-entering away mode" return 1 @@ -476,6 +504,7 @@ fm_afk_launch_start() { if daemon_lock_held_by_live_daemon; then fm_afk_launch_record_validate_if_present || return 1 + fm_afk_launch_versions_sweep if ! fm_afk_launch_flag_write; then fm_afk_launch_log "failed to refresh away-mode flag" return 1 @@ -484,20 +513,18 @@ fm_afk_launch_start() { return 0 fi - backup=$(mktemp -d "$FM_AFK_LAUNCH_STATE/.afk-launch-backup.XXXXXX") || return 1 if [ -f "$FM_AFK_LAUNCH_STATE/.afk" ]; then had_afk=1 - cp "$FM_AFK_LAUNCH_STATE/.afk" "$backup/.afk" || { rm -rf "$backup"; return 1; } + afk_content=$(cat "$FM_AFK_LAUNCH_STATE/.afk" 2>/dev/null) || return 1 fi - for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged; do - if [ -e "$FM_AFK_LAUNCH_STATE/$artifact" ]; then - cp -p "$FM_AFK_LAUNCH_STATE/$artifact" "$backup/$artifact" || { rm -rf "$backup"; return 1; } - fi - done + version=$(fm_afk_launch_entry_capture) || { + fm_afk_launch_log "could not capture the pre-launch away batch as an immutable version" + return 1 + } if ! fm_afk_launch_reconcile; then result=1 else - if fm_afk_clear_stale_artifacts "$FM_AFK_LAUNCH_STATE"; then + if fm_afk_clear_stale_artifacts "$FM_AFK_LAUNCH_STATE" "$had_afk"; then result=0 else fm_afk_launch_log "failed to clear stale away-mode artifacts" @@ -521,16 +548,19 @@ fm_afk_launch_start() { ;; esac fi + # On success the published version is deliberately RETAINED: this entry has + # just retired the previous session's unit from the live paths, so that version + # is the only surviving copy of any line it still owed captain chat. The return + # catch-up folds it and retires it, so nothing is discarded before + # acknowledgement. if [ "$result" -ne 0 ]; then - fm_afk_launch_restore_backup "$backup" "$had_afk" || result=1 - else - rm -rf "$backup" || result=1 + fm_afk_launch_rollback "$version" "$had_afk" "$afk_content" || result=1 fi return "$result" } fm_afk_launch_start_native() { - local backup artifact had_afk=0 result=0 + local version afk_content='' had_afk=0 result=0 mkdir -p "$FM_AFK_LAUNCH_STATE" || return 1 if [ -e "$FM_AFK_LAUNCH_STATE/.afk-return-catchup" ]; then fm_afk_launch_log "return catch-up is still pending; run bin/fm-afk-return.sh check before re-entering away mode" @@ -538,36 +568,39 @@ fm_afk_launch_start_native() { fi if daemon_lock_held_by_live_daemon; then fm_afk_launch_record_validate_if_present || return 1 + fm_afk_launch_versions_sweep fm_afk_launch_flag_write || return 1 fm_afk_launch_log "daemon already running; refreshed away-mode flag" return 0 fi - backup=$(mktemp -d "$FM_AFK_LAUNCH_STATE/.afk-launch-backup.XXXXXX") || return 1 if [ -f "$FM_AFK_LAUNCH_STATE/.afk" ]; then had_afk=1 - cp "$FM_AFK_LAUNCH_STATE/.afk" "$backup/.afk" || { rm -rf "$backup"; return 1; } + afk_content=$(cat "$FM_AFK_LAUNCH_STATE/.afk" 2>/dev/null) || return 1 fi - for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged; do - if [ -e "$FM_AFK_LAUNCH_STATE/$artifact" ]; then - cp -p "$FM_AFK_LAUNCH_STATE/$artifact" "$backup/$artifact" || { rm -rf "$backup"; return 1; } - fi - done + version=$(fm_afk_launch_entry_capture) || { + fm_afk_launch_log "could not capture the pre-launch away batch as an immutable version" + return 1 + } fm_afk_launch_reconcile || result=1 if [ "$result" -eq 0 ]; then - if ! fm_afk_clear_stale_artifacts "$FM_AFK_LAUNCH_STATE"; then + if ! fm_afk_clear_stale_artifacts "$FM_AFK_LAUNCH_STATE" "$had_afk"; then fm_afk_launch_log "failed to clear stale away-mode artifacts" result=1 - elif ! fm_afk_launch_flag_write; then - result=1 fi fi + if [ "$result" -eq 0 ] && ! fm_afk_launch_flag_write; then + result=1 + fi if [ "$result" -eq 0 ]; then fm_afk_launch_record_write none - native || result=1 fi + # On success the published version is deliberately RETAINED: this entry has + # just retired the previous session's unit from the live paths, so that version + # is the only surviving copy of any line it still owed captain chat. The return + # catch-up folds it and retires it, so nothing is discarded before + # acknowledgement. if [ "$result" -ne 0 ]; then - fm_afk_launch_restore_backup "$backup" "$had_afk" || result=1 - else - rm -rf "$backup" || result=1 + fm_afk_launch_rollback "$version" "$had_afk" "$afk_content" || result=1 fi return "$result" } diff --git a/bin/fm-afk-return.sh b/bin/fm-afk-return.sh index 316479852f..927e99c7fb 100755 --- a/bin/fm-afk-return.sh +++ b/bin/fm-afk-return.sh @@ -27,6 +27,10 @@ FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" GATE="$STATE/.afk-return-catchup" +# The away batch ledger owner: the return path folds and retires the away records +# through it rather than deleting them itself. +# shellcheck source=bin/fm-away-ledger-lib.sh +. "$SCRIPT_DIR/fm-away-ledger-lib.sh" LOCK="$STATE/.afk-return-catchup.lock" usage() { @@ -121,10 +125,21 @@ print_blockers() { # } clear_delivery_artifacts() { - rm -f \ - "$STATE/.subsuper-escalations" \ - "$STATE/.subsuper-escalations.since" \ - "$STATE/.subsuper-inject-wedged" + # Retire the complete owned unit and every away batch version through the + # ledger owner. This is the ACKNOWLEDGEMENT boundary: it only runs once the + # catch-up evidence has folded in the live digests and every retained version + # and no blocker is left, and it runs BEFORE the gate is removed - its status + # decides whether the gate may close at all. A version that survived retirement + # would otherwise be folded again by the next catch-up and presented as fresh + # work, so a failure here keeps the gate open instead. The text-free + # .status delivery evidence is deliberately kept. + local result=0 + away_ledger_retire_batch "$STATE" "$STATE/.subsuper-escalations" \ + "$STATE/.subsuper-inject-wedged" || result=1 + away_ledger_versions_retire_all "$STATE" || result=1 + [ "$result" -eq 0 ] \ + || printf 'fm-afk-return: some away batch records could not be retired after catch-up\n' >&2 + return "$result" } return_guard() { @@ -141,7 +156,8 @@ return_guard() { } return_reconcile() { - local evidence blockers drained wedge escalations lifecycle_ok=1 + local evidence blockers drained wedge escalations away_items lifecycle_ok=1 + local fold_retained_rc=0 evidence=$(mktemp "$STATE/.afk-return-evidence.XXXXXX") || return 1 blockers=$(mktemp "$STATE/.afk-return-blockers.XXXXXX") || { rm -f "$evidence"; return 1; } preserve_evidence "$evidence" @@ -160,32 +176,78 @@ return_reconcile() { } append_evidence wake "$drained" "$evidence" - if [ -s "$STATE/.subsuper-inject-wedged" ]; then - wedge=$(head -1 "$STATE/.subsuper-inject-wedged" 2>/dev/null || true) - append_evidence wedge "$wedge" "$evidence" - fi - if [ -s "$STATE/.subsuper-escalations" ]; then - escalations=$(cat "$STATE/.subsuper-escalations" 2>/dev/null || true) - append_evidence escalation "$escalations" "$evidence" + # Every catch-up read below appends into the SAME evidence kind whether the text + # came from the live unit or from a retained immutable batch version, because a + # retained version's content is a prefix-subset of the live unit whenever the + # live unit was materialised from it. append_evidence dedupes exact records per + # kind, so each logical escalation reaches Firstmate exactly once instead of + # once per source under two different labels. + wedge=$( + if [ -s "$STATE/.subsuper-inject-wedged" ]; then + head -1 "$STATE/.subsuper-inject-wedged" 2>/dev/null || true + fi + away_ledger_fold_versions "$STATE" wedges 2>/dev/null || true + ) + append_evidence wedge "$wedge" "$evidence" + # Only the lines the ledger has NOT already accounted for in a private away + # digest: anything at or below `accounted` is reported once below as + # away-telegram, and appending the whole buffer here would present the same + # escalation to Firstmate as two separate catch-up items to act on. An absent + # or unreadable record falls back to the whole buffer, so an unusable ledger + # over-reports rather than dropping a captain-relevant line. + # + # The retained side is every immutable away batch version still in the store: + # one whose active-pointer switch was refused (a daemon still live, e.g. after + # a failed stop), one whose launch transaction was abandoned, and the one a + # fresh away entry published before retiring that content from the live paths. + # All of it is read-only here; nothing is retired before the acknowledgement in + # clear_delivery_artifacts. + escalations=$( + away_ledger_unaccounted_lines "$STATE/.subsuper-escalations" 2>/dev/null || true + away_ledger_fold_versions "$STATE" escalations 2>/dev/null || exit 1 + ) || fold_retained_rc=$? + append_evidence escalation "$escalations" "$evidence" + # An accepted away batch has already been receipted and its buffer truncated, so + # the ledger owner's private digests are the only local copy of those events. + # Fold them in BEFORE anything retires them. + away_items=$( + away_ledger_fold_digests "$STATE" 2>/dev/null || true + away_ledger_fold_versions "$STATE" digests 2>/dev/null || exit 1 + ) || fold_retained_rc=$? + append_evidence away-telegram "$away_items" "$evidence" + if [ "$fold_retained_rc" -ne 0 ]; then + append_evidence lifecycle 'a retained away-mode batch version could not be read; retry catch-up before ordinary work' "$evidence" + lifecycle_ok=0 fi scan_open_blockers > "$blockers" - if [ "$lifecycle_ok" -ne 1 ] || [ -s "$blockers" ]; then + # This pass's folded evidence is made durable in the gate BEFORE anything is + # retired, so a gate write that fails can never leave freshly folded escalation + # text deleted from disk and recorded nowhere. Only once the gate holds it is + # retirement attempted, and retirement is itself part of acknowledgement: a + # failure keeps the gate open exactly like an unresolved blocker. Re-running is + # safe and does not duplicate - the gate preserves this evidence and + # append_evidence dedupes exact records, so the next pass folds the same + # surviving version into the same records it already published. + write_gate "$evidence" "$blockers" || { rm -f "$evidence" "$blockers"; return 1; } + if [ "$lifecycle_ok" -eq 1 ] && [ ! -s "$blockers" ]; then + if clear_delivery_artifacts; then + print_evidence "$evidence" + rm -f "$GATE" + rm -f "$evidence" "$blockers" + printf 'fm-afk-return: catch-up clear; ordinary captain work may proceed\n' + return 0 + fi + append_evidence lifecycle 'away batch records could not be retired; retry catch-up before ordinary work' "$evidence" + lifecycle_ok=0 write_gate "$evidence" "$blockers" || { rm -f "$evidence" "$blockers"; return 1; } - printf 'fm-afk-return: catch-up must finish before the captain request\n' >&2 - print_evidence "$GATE" >&2 - print_blockers "$GATE" >&2 - printf 'fm-afk-return: handle each blocker now, or close it with resolved [key=...] and append a durable reclassification reason, then run bin/fm-afk-return.sh check\n' >&2 - rm -f "$evidence" "$blockers" - return 3 fi - - print_evidence "$evidence" - rm -f "$GATE" - clear_delivery_artifacts + printf 'fm-afk-return: catch-up must finish before the captain request\n' >&2 + print_evidence "$GATE" >&2 + print_blockers "$GATE" >&2 + printf 'fm-afk-return: handle each blocker now, or close it with resolved [key=...] and append a durable reclassification reason, then run bin/fm-afk-return.sh check\n' >&2 rm -f "$evidence" "$blockers" - printf 'fm-afk-return: catch-up clear; ordinary captain work may proceed\n' - return 0 + return 3 } main() { diff --git a/bin/fm-afk-start.sh b/bin/fm-afk-start.sh index 532d57b7ce..604ebe296d 100755 --- a/bin/fm-afk-start.sh +++ b/bin/fm-afk-start.sh @@ -42,6 +42,11 @@ FM_AFK_DAEMON="$FM_AFK_START_DIR/fm-supervise-daemon.sh" # shellcheck source=bin/fm-wake-lib.sh . "$FM_AFK_START_DIR/fm-wake-lib.sh" +# The away batch ledger owner: this script retires a previous session's ledger +# and working records through it instead of deleting those files itself. +# shellcheck source=bin/fm-away-ledger-lib.sh +. "$FM_AFK_START_DIR/fm-away-ledger-lib.sh" + fm_afk_start_usage() { sed -n '2,14p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' } @@ -59,11 +64,20 @@ fm_afk_start_usage() { # lifecycle" and bin/fm-supervise-daemon.sh's escalate_add/inject_wedge_alarm). # NOT called on a refresh (daemon already alive), so the current session's own # buffered escalations are preserved. -fm_afk_clear_stale_artifacts() { # - local state=$1 - rm -f "$state/.subsuper-escalations" \ - "$state/.subsuper-escalations.since" \ - "$state/.subsuper-inject-wedged" 2>/dev/null +# +# , when 1, means state/.afk was already present before this call - +# a crashed-daemon restart or a failed re-entry mid-away-session, not a +# genuinely fresh entry - so the digest directory is preserved rather than +# retired: those items may be accepted-and-receipted but not yet consumed or +# folded by Firstmate, and this path never re-derives them the way the +# escalation buffer itself is re-derived by the daemon's heartbeat scan. +fm_afk_clear_stale_artifacts() { # [had-afk] + local state=$1 had_afk=${2:-0} + # The ledger owner retires the complete owned unit (buffer, wedge marker, + # sidecar, working records) itself; the text-free .status delivery + # evidence is deliberately left in place. + away_ledger_retire_batch "$state" "$state/.subsuper-escalations" \ + "$state/.subsuper-inject-wedged" "$had_afk" } daemon_lock_owner() { @@ -118,6 +132,8 @@ fm_afk_start_main() { esac mkdir -p "$FM_AFK_STATE" + local had_afk=0 + [ -f "$FM_AFK_STATE/.afk" ] && had_afk=1 if [ "${FM_AFK_STATE_PREPARED:-0}" = 1 ]; then [ -f "$FM_AFK_STATE/.afk" ] || { echo "afk: launcher-prepared state is missing" >&2; return 1; } else @@ -137,8 +153,41 @@ fm_afk_start_main() { # Fresh start: clear the previous away session's stale delivery artifacts # before the new daemon can surface them (fix for the leaked-artifact defect). + # + # A direct start is an away entry like any other, so it goes through the SAME + # owner-controlled entry-boundary transaction the launcher uses: sweep, replay + # an interrupted active-pointer switch, and capture the predecessor unit as one + # immutable version FIRST. Without that capture this clear would delete a + # crashed session's un-flushed escalation lines with no surviving copy, so a + # failed capture refuses the start instead of clearing anything. if [ "${FM_AFK_STATE_PREPARED:-0}" != 1 ]; then - fm_afk_clear_stale_artifacts "$FM_AFK_STATE" + if ! away_ledger_entry_capture "$FM_AFK_STATE" \ + "$FM_AFK_STATE/.subsuper-escalations" \ + "$FM_AFK_STATE/.subsuper-inject-wedged" >/dev/null; then + echo "afk: could not capture the prior away batch as an immutable version; refusing to clear it" >&2 + # Away mode never actually started, so roll back the flag THIS invocation + # wrote rather than leaving the system flagged away with no daemon + # supervising it. A flag that predates this call belongs to the session + # already in progress and is left alone. + if [ "$had_afk" -eq 0 ]; then + rm -f "$FM_AFK_STATE/.afk" 2>/dev/null || true + fi + return 1 + fi + # The clear is an owner transaction too, so it can legitimately refuse (owner + # lock busy, a live artifact that will not go). Under this script's errexit + # an unguarded call would end the run silently, mid-retirement, with away + # mode already flagged and no daemon. Fail honestly instead: the captured + # version above retains every owed line, so nothing is lost. + local clear_rc=0 + fm_afk_clear_stale_artifacts "$FM_AFK_STATE" "$had_afk" || clear_rc=$? + if [ "$clear_rc" -ne 0 ]; then + echo "afk: could not retire the prior away batch through the ledger owner (its immutable version is retained); refusing to start the daemon" >&2 + if [ "$had_afk" -eq 0 ]; then + rm -f "$FM_AFK_STATE/.afk" 2>/dev/null || true + fi + return 1 + fi fi echo "afk: starting supervise daemon in foreground; keep this command as a tracked background session" diff --git a/bin/fm-away-ledger-lib.sh b/bin/fm-away-ledger-lib.sh new file mode 100644 index 0000000000..17274e89f2 --- /dev/null +++ b/bin/fm-away-ledger-lib.sh @@ -0,0 +1,1470 @@ +# shellcheck shell=bash +# The single owner of the away-mode escalation batch ledger. +# +# One away batch is the buffer state/.subsuper-escalations plus ONE ledger record +# in its state/.subsuper-escalations.since sidecar: +# +# - +# +# arrival-epoch keeps the batch-age semantics the daemon's batching needs. +# nonce makes the identity minted-once, so no later batch can ever +# inherit a retired batch's counts. +# reserved leading buffer lines that must NEVER be sent to the phone +# again. Claimed BEFORE the network call, so a crash mid-send +# cannot become a duplicate captain alert. A reservation is given +# back ONLY when the send provably never left the box. +# confirmed leading lines Telegram provably accepted, proven by the durable +# delivery evidence - never inferred from a reservation, and only +# ever advanced over a CONTIGUOUS prefix (away_ledger_confirm +# refuses to jump an unconfirmed gap), so an accepted suffix can +# never imply acceptance of an earlier uncertain line. +# accounted leading lines whose text is captured in a private away digest. +# Anything past this count still has to reach captain chat. +# attempt the batch's delivery-attempt ordinal, folded into every delivery +# id. It advances ONLY when a proven-local failure retires an +# attempt, which is what lets the same lines be offered to the +# phone again under a fresh id while the retired attempt's +# evidence stays durable and auditable. An ambiguous outcome +# never advances it, so it can never be retried. +# retry-after the epoch before which this batch may not be offered to the +# phone again. Set when a proven-local failure retires an attempt, +# so a persistent local failure retries on a bounded schedule +# instead of once per housekeeping tick; cleared by acceptance. +# delivery-id the most recent delivery this batch attempted. +# +# Earlier record shapes - the bare arrival epoch of the pre-ledger form, and the +# five- and six-field forms that predate the attempt ordinal and the retry +# schedule - are normalised by away_ledger_parse and migrated in place by the LIVE +# read (away_ledger_read), so an in-place daemon upgrade during a live away +# session keeps its counts instead of failing closed and repeating +# already-delivered lines in captain chat. A read of a published immutable version +# goes through away_ledger_peek instead, which never writes; that migration +# happens only while a successor version is being constructed. +# +# The invariant accounted <= confirmed <= reserved holds at every transition, and +# an unparseable record is reported as `unknown`, which callers must treat as +# fail-closed: nothing to the phone, everything to the visible in-session +# escalation. Callers never keep their own counters and never retire a digest or +# a ledger record themselves - they query and transition this owner, so the away +# daemon, away start, and away return all agree on one state machine. +# +# All of it lives in the sidecar because every away lifecycle path already +# retires, publishes, and restores that sidecar together with its buffer through +# this owner's immutable version store (away_ledger_version_publish / +# away_ledger_version_activate / away_ledger_retire_batch), never by enumerating +# the artifact set itself. away launch's crash rollback publishes the whole unit +# (buffer, sidecar, wedge marker, digest directory) as one opaque immutable +# version rather than reading or advancing counts, so it cannot desynchronise the +# state machine, and away-mode's own flag file is the one artifact outside this +# unit - it belongs to the launcher, not the ledger. + +FM_AWAY_LEDGER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=bin/fm-x-lib.sh +. "$FM_AWAY_LEDGER_DIR/fm-x-lib.sh" + +away_ledger_now() { date +%s; } + +# One shared numeric-count validator, so every entry point rejects a malformed or +# missing count on its own rather than concatenating several and hoping. +away_ledger_is_count() { # ... + local v + for v in "$@"; do + case "$v" in + ''|*[!0-9]*) return 1 ;; + esac + done + return 0 +} + +away_ledger_hash() { + local h + if command -v md5 >/dev/null 2>&1; then h=$(printf '%s' "$1" | md5 -q) + else h=$(printf '%s' "$1" | md5sum | cut -d ' ' -f1); fi + printf '%s\n' "${h:0:16}" +} + +away_ledger_digest_dir() { # + printf '%s\n' "$1/tg-away-digest" +} + +away_ledger_delivery_dir() { # + printf '%s\n' "$1/tg-away-delivery" +} + +# A minted-once identity. The entropy comes from a mktemp token rather than +# $RANDOM: this owner mints ids from inside command substitutions, and in bash 3.2 +# $RANDOM restarts from the parent's state in every one of those, so two mints in +# the same second within one process produce the SAME value - which would give two +# batches one identity and two versions one directory name. mktemp's uniqueness +# does not depend on that sequence. , when given, is where the token is +# created (it is removed immediately); the old scheme remains the fallback. +away_ledger_mint_id() { # [scope-dir] + local now token dir + now=$(away_ledger_now) + dir=${1:-${TMPDIR:-/tmp}} + if token=$(umask 077; mktemp "$dir/.mint.XXXXXX" 2>/dev/null); then + printf '%s-%s\n' "$now" "$(away_ledger_hash "$$-${token##*/}-${RANDOM:-0}-$now")" + rm -f -- "$token" 2>/dev/null + return 0 + fi + printf '%s-%s\n' "$now" "$(away_ledger_hash "$$-${RANDOM:-0}-${RANDOM:-0}-$now")" +} + +# Write one record next to with a same-directory mktemp plus mv, so a reader +# never sees a half-written ledger. PRIVATE: this is the raw file write used to +# build a successor version's own sidecar inside private staging. A LIVE record +# change goes through away_ledger_write, which commits a validated successor +# version and switches the active pointer before any live path changes. +_away_ledger_record_put() { # + local buf=$1 id=$2 reserved=$3 confirmed=$4 accounted=$5 attempt=$6 retry=$7 did=$8 file tmp + away_ledger_is_count "$reserved" "$confirmed" "$accounted" "$attempt" "$retry" || return 1 + file="${buf}.since" + tmp=$(umask 077; mktemp "${file}.XXXXXX" 2>/dev/null) || return 1 + if ! printf '%s %s %s %s %s %s %s\n' \ + "$id" "$reserved" "$confirmed" "$accounted" "$attempt" "$retry" "$did" \ + > "$tmp" 2>/dev/null \ + || ! mv -f "$tmp" "$file" 2>/dev/null; then + rm -f -- "$tmp" + return 1 + fi + return 0 +} + +# The live away unit a buffer path belongs to. The unit is one fixed artifact set +# inside one state directory, so the transition path derives it from the buffer +# rather than making every caller thread three paths through; the lifecycle +# callers keep the explicit-path API. +away_ledger_state_of() { # + printf '%s\n' "${1%/*}" +} + +away_ledger_wedge_of() { # + printf '%s\n' "$1/.subsuper-inject-wedged" +} + +# The in-session (captain chat) delivery state of the current batch, part of the +# owned unit like every other artifact: " attempting|confirmed". +# It exists so a bookkeeping failure AFTER a confirmed submit can never become a +# second delivery of the same batch into captain chat. +away_ledger_chat_of() { # + printf '%s\n' "$1/.subsuper-chat-delivery" +} + +# The unit's artifact slots, in one place, so a slot can never be copied by the +# staging build and then forgotten by the projection (or the reverse). +_away_ledger_unit_paths() { # + printf '%s\n%s\n%s\n%s\n' "$2" "${2}.since" "$3" "$(away_ledger_chat_of "$1")" +} + +_away_ledger_unit_slots() { + printf 'escalations\nescalations.since\nwedge\nchat\n' +} + +# Every LIVE record change is a version transition: a complete successor version +# carrying the new record is built, validated, and published, and only then does +# the active pointer switch and the live projection get rewritten. +away_ledger_write() { # + local buf=$1 state + away_ledger_is_count "$3" "$4" "$5" "$6" "$7" || return 1 + state=$(away_ledger_state_of "$buf") + away_ledger_transact "$state" "$buf" "$(away_ledger_wedge_of "$state")" \ + _away_ledger_mut_record "$2" "$3" "$4" "$5" "$6" "$7" "$8" +} + +# Start the ledger for a brand new batch: a fresh identity and nothing delivered. +away_ledger_open() { # + away_ledger_write "$1" "$(away_ledger_mint_id "$(away_ledger_state_of "$1")")" 0 0 0 0 0 none +} + +# Append one distilled escalation line to the batch. The daemon's own append is a +# version transition like every other, so the line is durable in a published +# version before it is visible in the live buffer. +away_ledger_append() { # + local buf=$1 state + state=$(away_ledger_state_of "$buf") + away_ledger_transact "$state" "$buf" "$(away_ledger_wedge_of "$state")" \ + _away_ledger_mut_append "$2" +} + +# Retire a fully in-session-delivered batch: the buffer is emptied and the record +# and wedge marker are dropped, all as ONE successor version, so no intermediate +# state where a truncated buffer is still paired with its old counts is ever +# published or projected. +away_ledger_truncate() { # + local buf=$1 state + state=$(away_ledger_state_of "$buf") + away_ledger_transact "$state" "$buf" "$(away_ledger_wedge_of "$state")" \ + _away_ledger_mut_truncate +} + +# Record the wedge evidence for the unit, text on stdin only. +away_ledger_wedge_record() { # + local state=$1 tmp rc + tmp=$(umask 077; mktemp "$state/.subsuper-inject-wedged.stage.XXXXXX" 2>/dev/null) || return 1 + if ! cat > "$tmp" 2>/dev/null; then + rm -f -- "$tmp" 2>/dev/null + return 1 + fi + away_ledger_transact "$state" "$state/.subsuper-escalations" \ + "$(away_ledger_wedge_of "$state")" _away_ledger_mut_wedge "$tmp" + rc=$? + rm -f -- "$tmp" 2>/dev/null + return "$rc" +} + +away_ledger_wedge_clear() { # + local state=$1 + [ -e "$(away_ledger_wedge_of "$state")" ] || return 0 + away_ledger_transact "$state" "$state/.subsuper-escalations" \ + "$(away_ledger_wedge_of "$state")" _away_ledger_mut_wedge_clear +} + +# The transition mutators. Each edits ONLY the private staging copy of the unit it +# is handed; none may touch a live path, and the transaction below is what turns a +# mutated staging directory into a published version and then a live projection. +_away_ledger_mut_record() { # + local staging=$8 + [ -e "$staging/escalations" ] || : > "$staging/escalations" 2>/dev/null || return 1 + _away_ledger_record_put "$staging/escalations" "$1" "$2" "$3" "$4" "$5" "$6" "$7" +} + +_away_ledger_mut_append() { # + printf '%s\n' "$1" >> "$2/escalations" 2>/dev/null +} + +_away_ledger_mut_truncate() { # + : > "$1/escalations" 2>/dev/null || return 1 + rm -f -- "$1/escalations.since" "$1/wedge" "$1/chat" 2>/dev/null || return 1 + return 0 +} + +_away_ledger_mut_chat() { # + printf '%s %s %s %s\n' "$1" "$2" "$3" "$4" > "$5/chat" 2>/dev/null +} + +# The in-session delivery receipt, parsed as INDEPENDENT fields: +# +# +# +# `confirmed` is the monotonic prefix of the batch captain chat PROVABLY has, and +# the attempt range is the suffix a submit is currently offering it. They are +# separate on purpose: starting an attempt must never lower the confirmed prefix, +# so an inject that fails (or a crash between the attempt record and the submit) +# cannot drag already-delivered lines back into captain chat. +# +# The legacy 3-field " attempting|confirmed" shape is read forward: +# `confirmed` there counted only for the confirmed phase, and an `attempting` +# record proved nothing. +_away_ledger_chat_parse() { # -> " " + local id a b c extra + IFS=' ' read -r id a b c extra \ + < <(head -n 1 "$(away_ledger_chat_of "$1")" 2>/dev/null || true) + [ -z "${extra:-}" ] || { printf 'none 0 0 0\n'; return 0; } + case "${id:-}" in + ''|*[!0-9a-zA-Z-]*) printf 'none 0 0 0\n'; return 0 ;; + esac + if [ -n "${c:-}" ]; then + if away_ledger_is_count "${a:-}" "${b:-}" "$c"; then + printf '%s %s %s %s\n' "$id" "$a" "$b" "$c" + return 0 + fi + elif away_ledger_is_count "${a:-}"; then + case "${b:-}" in + confirmed) printf '%s %s 0 0\n' "$id" "$a"; return 0 ;; + attempting) printf '%s 0 0 %s\n' "$id" "$a"; return 0 ;; + esac + fi + printf 'none 0 0 0\n' +} + +# The identity the STORE itself can vouch for: the batch named by the validated +# active version pointer. It is read from the pointer and its completeness proof, +# never from the (possibly unreadable) ledger payload or from message text, so it +# stays available exactly when the payload cannot be trusted. The `v-` prefix +# keeps its provenance explicit and cannot collide with a minted ledger id, which +# always starts with an epoch digit. +_away_ledger_chat_version_id() { # + local dir name + dir=$(away_ledger_versions_dir "$1") + name=$(away_ledger_pointer_read "$dir/active") || return 1 + away_ledger_version_is_complete "$dir/$name" || return 1 + name=${name#v.} + case "$name" in + ''|*[!0-9a-zA-Z-]*) return 1 ;; + esac + printf 'v-%s\n' "$name" +} + +# The receipt names its own batch, so it stays usable when the ledger sidecar +# itself is unreadable: a caller with no id (or the placeholder `none`) adopts +# the recorded identity rather than clobbering it with a nameless record, and a +# batch whose payload went unreadable BEFORE it ever had a receipt falls back to +# the active version's identity so the confirmed prefix can still be opened. +_away_ledger_chat_effective_id() { # + case "${3:-}" in + ''|none) ;; + *) printf '%s\n' "$3"; return 0 ;; + esac + case "${2:-}" in + ''|none) ;; + *) printf '%s\n' "$2"; return 0 ;; + esac + _away_ledger_chat_version_id "$1" || printf 'none\n' + return 0 +} + +# Open an attempt on the (from, to] suffix. The confirmed prefix is carried +# forward untouched for the same batch, so this transition can only ever add +# information about what is in flight. +away_ledger_chat_mark_attempt() { # + local state=$1 id=$2 from=$3 to=$4 rec rid rconfirmed confirmed + away_ledger_is_count "$from" "$to" || return 1 + rec=$(_away_ledger_chat_parse "$state") + IFS=' ' read -r rid rconfirmed _ _ <<< "$rec" + id=$(_away_ledger_chat_effective_id "$state" "$rid" "$id") + case "$id" in + ''|none|*[!0-9a-zA-Z-]*) return 1 ;; + esac + confirmed=0 + [ "$id" = "$rid" ] && confirmed=$rconfirmed + [ "$from" -ge "$confirmed" ] || from=$confirmed + away_ledger_transact "$state" "$state/.subsuper-escalations" \ + "$(away_ledger_wedge_of "$state")" _away_ledger_mut_chat "$id" "$confirmed" "$from" "$to" +} + +# A verified submit advances the confirmed prefix and closes the attempt. The +# advance is monotonic: a stale or smaller count can never walk it back. +away_ledger_chat_mark_confirmed() { # + local state=$1 id=$2 to=$3 rec rid rconfirmed confirmed + away_ledger_is_count "$to" || return 1 + rec=$(_away_ledger_chat_parse "$state") + IFS=' ' read -r rid rconfirmed _ _ <<< "$rec" + id=$(_away_ledger_chat_effective_id "$state" "$rid" "$id") + case "$id" in + ''|none|*[!0-9a-zA-Z-]*) return 1 ;; + esac + confirmed=$to + if [ "$id" = "$rid" ] && [ "$rconfirmed" -gt "$confirmed" ]; then + confirmed=$rconfirmed + fi + away_ledger_transact "$state" "$state/.subsuper-escalations" \ + "$(away_ledger_wedge_of "$state")" _away_ledger_mut_chat "$id" "$confirmed" 0 0 +} + +# How many lines of captain chat PROVABLY already received. A batch +# only ever grows, so the prefix is monotonic and a later flush may only ever +# offer chat the lines PAST it. is optional: passing nothing (or the +# placeholder `none` a caller uses when the sidecar is unreadable) answers for +# whichever batch the receipt itself names, so an unreadable ledger over-reports +# only the lines chat has NOT been shown. +away_ledger_chat_delivered() { # [batch-id] + local rec rid rconfirmed + rec=$(_away_ledger_chat_parse "$1") + IFS=' ' read -r rid rconfirmed _ _ <<< "$rec" + [ "$rid" != none ] || { printf '0\n'; return 0; } + case "${2:-}" in + ''|none) ;; + "$rid") ;; + *) printf '0\n'; return 0 ;; + esac + printf '%s\n' "$rconfirmed" +} + +_away_ledger_mut_wedge() { # + cp -p "$1" "$2/wedge" 2>/dev/null +} + +_away_ledger_mut_wedge_clear() { # + rm -f -- "$1/wedge" 2>/dev/null +} + +# Capture the lines in (, ] into this delivery's private digest inside +# the successor version and advance the accounted count in the SAME version, so a +# digested line and the count that accounts for it are never separately visible. +_away_ledger_mut_digest() { # + local did=$1 from=$2 to=$3 staging=$4 lines rec migrated id reserved confirmed accounted attempt retry current + [ -s "$staging/escalations" ] || return 0 + lines=$(( $(wc -l < "$staging/escalations" 2>/dev/null || echo 0) )) + [ "$to" -le "$lines" ] || return 1 + [ "$to" -gt "$from" ] || return 0 + (umask 077; mkdir -p "$staging/tg-away-digest" 2>/dev/null) || return 1 + if ! (umask 077; sed -n "$((from + 1)),${to}p" "$staging/escalations" \ + > "$staging/tg-away-digest/$did.items") 2>/dev/null; then + return 1 + fi + rec=$(away_ledger_parse "$staging/escalations") + [ "$rec" != unknown ] || return 1 + IFS=' ' read -r migrated id reserved confirmed accounted attempt retry current <<< "$rec" + accounted=$to + [ "$accounted" -le "$confirmed" ] || accounted=$confirmed + _away_ledger_record_put "$staging/escalations" "$id" "$reserved" "$confirmed" \ + "$accounted" "$attempt" "$retry" "$did" +} + +# PURE parse: print " +# ", or the single word +# `unknown` when the record is absent, malformed, or violates the +# accounted <= confirmed <= reserved invariant. Writes NOTHING, so it is the only +# record reader safe to point at a published immutable version. Every earlier +# record shape is normalised here and flagged as needing migration rather than +# rejected: +# 1 field the pre-ledger bare arrival epoch -> minted identity, zero counts +# 5 fields id reserved confirmed accounted did -> attempt 0, retry-after 0 +# 6 fields ... attempt did -> retry-after 0 +away_ledger_parse() { # + local buf=$1 raw f1 f2 f3 f4 f5 f6 f7 extra + local id reserved confirmed accounted attempt retry did epoch nonce migrated=0 + raw=$(head -n 1 "${buf}.since" 2>/dev/null || true) + IFS=' ' read -r f1 f2 f3 f4 f5 f6 f7 extra <<< "$raw" + if [ -z "${f1:-}" ] || [ -n "${extra:-}" ]; then printf 'unknown\n'; return 0; fi + id=$f1; reserved=${f2:-}; confirmed=${f3:-}; accounted=${f4:-} + if [ -z "${f2:-}" ]; then + reserved=0; confirmed=0; accounted=0; attempt=0; retry=0; did=none; migrated=1 + elif [ -z "${f3:-}" ] || [ -z "${f4:-}" ] || [ -z "${f5:-}" ]; then + printf 'unknown\n'; return 0 + elif [ -z "${f6:-}" ]; then + attempt=0; retry=0; did=$f5; migrated=1 + elif [ -z "${f7:-}" ]; then + attempt=$f5; retry=0; did=$f6; migrated=1 + else + attempt=$f5; retry=$f6; did=$f7 + fi + epoch=${id%%-*} + nonce=${id#*-} + away_ledger_is_count "$epoch" "$reserved" "$confirmed" "$accounted" "$attempt" "$retry" \ + || { printf 'unknown\n'; return 0; } + case "$did" in ''|*[!0-9a-zA-Z-]*) printf 'unknown\n'; return 0 ;; esac + if [ "$accounted" -gt "$confirmed" ] || [ "$confirmed" -gt "$reserved" ]; then + printf 'unknown\n'; return 0 + fi + if [ -z "$nonce" ] || [ "$nonce" = "$id" ]; then + id="$epoch-$(away_ledger_hash "$$-${RANDOM:-0}-$epoch")" + migrated=1 + fi + printf '%s %s %s %s %s %s %s %s\n' \ + "$migrated" "$id" "$reserved" "$confirmed" "$accounted" "$attempt" "$retry" "$did" +} + +# The read-only record view: the normalised record with no migration write at +# all. Every reader that must not mutate its subject - the return catch-up's fold +# of a published immutable version above all - goes through this, never through +# away_ledger_read. +away_ledger_peek() { # + local rec + rec=$(away_ledger_parse "$1") + [ "$rec" != unknown ] || { printf 'unknown\n'; return 0; } + printf '%s\n' "${rec#* }" +} + +# The LIVE record view: parse, and migrate an earlier record shape in place so an +# in-place daemon upgrade during a live away session keeps its counts. Only ever +# pointed at a live buffer, never at a version directory - a published version is +# immutable, and a successor version is where a migrated record belongs. +away_ledger_read() { # + local buf=$1 rec migrated id reserved confirmed accounted attempt retry did + rec=$(away_ledger_parse "$buf") + [ "$rec" != unknown ] || { printf 'unknown\n'; return 0; } + IFS=' ' read -r migrated id reserved confirmed accounted attempt retry did <<< "$rec" + if [ "$migrated" -eq 1 ]; then + away_ledger_write "$buf" "$id" "$reserved" "$confirmed" "$accounted" "$attempt" "$retry" "$did" \ + || { printf 'unknown\n'; return 0; } + fi + printf '%s %s %s %s %s %s %s\n' \ + "$id" "$reserved" "$confirmed" "$accounted" "$attempt" "$retry" "$did" +} + +# The batch's arrival epoch, for age-based batching. Non-zero when there is no +# usable record. +away_ledger_epoch() { # + local raw epoch + raw=$(head -n 1 "${1}.since" 2>/dev/null || true) + epoch=${raw%% *} + epoch=${epoch%%-*} + case "$epoch" in ''|*[!0-9]*) return 1 ;; esac + printf '%s\n' "$epoch" + return 0 +} + +# The only mutation path for the three counts. Reads the current record, applies +# ONE named transition, re-establishes the invariant, and writes it back, so no +# caller can leave the ledger internally inconsistent. +away_ledger_transition() { # reserved|confirmed|accounted [delivery-id] + local buf=$1 field=$2 value=$3 did=${4:-} + local rec id reserved confirmed accounted attempt retry current + away_ledger_is_count "$value" || return 1 + rec=$(away_ledger_read "$buf") + [ "$rec" != unknown ] || return 1 + IFS=' ' read -r id reserved confirmed accounted attempt retry current <<< "$rec" + [ -n "$did" ] || did=$current + case "$field" in + reserved) reserved=$value ;; + confirmed) confirmed=$value; retry=0 ;; + accounted) accounted=$value ;; + *) return 1 ;; + esac + [ "$confirmed" -le "$reserved" ] || confirmed=$reserved + [ "$accounted" -le "$confirmed" ] || accounted=$confirmed + away_ledger_write "$buf" "$id" "$reserved" "$confirmed" "$accounted" "$attempt" "$retry" "$did" +} + +# Whether the batch may be offered to the phone right now: false while a retired +# proven-local attempt's retry schedule has not yet elapsed. +away_ledger_retry_due() { # + local retry=${1:-0} + away_ledger_is_count "$retry" || return 1 + [ "$retry" -eq 0 ] && return 0 + [ "$(away_ledger_now)" -ge "$retry" ] +} + +# Claim lines for the phone BEFORE the network call. +away_ledger_reserve() { # + away_ledger_transition "$1" reserved "$2" "$3" +} + +# Retire an attempt that PROVABLY never left the box: give its claim back, advance +# the attempt ordinal, and arm the retry schedule, all in the same atomic write, so +# the very same lines can be offered to the phone again under a fresh delivery id +# once has elapsed, while the retired attempt's evidence record +# stays durable and auditable. This is the only transition that advances the +# ordinal, so an ambiguous outcome - which keeps its claim - can never be retried, +# and the schedule keeps a persistent local failure from re-attempting (and minting +# a fresh evidence record) on every housekeeping tick. +away_ledger_release() { # [retry-delay-secs] + local buf=$1 reserved=$2 did=$3 delay=${4:-0} + local rec id c a attempt retry current + away_ledger_is_count "$reserved" || return 1 + away_ledger_is_count "$delay" || delay=0 + rec=$(away_ledger_read "$buf") + [ "$rec" != unknown ] || return 1 + IFS=' ' read -r id _ c a attempt retry current <<< "$rec" + [ -n "$did" ] || did=$current + [ "$reserved" -ge "$c" ] || reserved=$c + retry=0 + [ "$delay" -le 0 ] || retry=$(( $(away_ledger_now) + delay )) + away_ledger_write "$buf" "$id" "$reserved" "$c" "$a" "$((attempt + 1))" "$retry" "$did" +} + +# Record evidence-proven Telegram acceptance for the range (, ]. +# Refuses unless is exactly the current confirmed count, so confirmation can +# only ever extend a contiguous proven prefix. `reserved > confirmed` therefore +# means an unresolved send, and the caller must not offer this batch to the phone +# again until that reservation is resolved. +away_ledger_confirm() { # + local buf=$1 from=$2 to=$3 did=$4 rec id reserved confirmed accounted attempt retry current + away_ledger_is_count "$from" "$to" || return 1 + rec=$(away_ledger_read "$buf") + [ "$rec" != unknown ] || return 1 + IFS=' ' read -r id reserved confirmed accounted attempt retry current <<< "$rec" + [ "$from" -eq "$confirmed" ] || return 1 + [ "$to" -ge "$confirmed" ] || return 1 + away_ledger_transition "$buf" confirmed "$to" "$did" +} + +# The delivery identity of one send: batch, attempt ordinal, first line sent, and +# the exact body. Built here so no caller mints an away-delivery id of its own, and +# so a retired proven-local attempt gets a genuinely new id instead of colliding +# with its own tombstoned evidence. +away_ledger_delivery_id() { # + away_ledger_is_count "$2" "$3" || return 1 + printf '%s-a%s-%s-%s\n' "$1" "$2" "$3" "$(away_ledger_hash "$4")" +} + +# Capture the batch lines in (, ] into this delivery's private digest +# and account for them. is the ACCOUNTED position rather than the send +# offset, so a delivery whose digest failed earlier is picked up by the next digest +# that succeeds; is the CONFIRMED count, so an unproven line is never recorded +# as delivered. +away_ledger_digest_record() { # + local state=$1 buf=$2 did=$3 from=$4 to=$5 + away_ledger_is_count "$from" "$to" || return 1 + [ -s "$buf" ] || return 0 + [ "$to" -gt "$from" ] || return 0 + away_ledger_transact "$state" "$buf" "$(away_ledger_wedge_of "$state")" \ + _away_ledger_mut_digest "$did" "$from" "$to" +} + +# Every delivery of a batch shares the batch id prefix, so ONE pointer covers all +# of a grown batch's digests - the receipt never names a single delivery's file. +away_ledger_digest_pointer() { # + printf 'state/tg-away-digest/%s-*.items\n' "$1" +} + +# Print every recorded away item so the return catch-up can fold them into its +# evidence BEFORE anything retires them. +away_ledger_fold_digests() { # + local state=$1 dir f + dir=$(away_ledger_digest_dir "$state") + [ -d "$dir" ] || return 0 + for f in "$dir"/*.items; do + [ -f "$f" ] || continue + cat "$f" 2>/dev/null || true + done + return 0 +} + +# Private 0700-directory spool for one outbound alert, so the text never lands +# outside the firstmate home even if the daemon is killed mid-send. +away_ledger_spool_create() { # + local state=$1 dir + dir=$(away_ledger_delivery_dir "$state") + fmx_private_artifact_dir_prepare "$dir" >/dev/null 2>&1 || return 1 + (umask 077; mktemp "$dir/.spool.XXXXXX" 2>/dev/null) || return 1 +} + +# Retire the ledger of a batch that has been fully delivered in session. +away_ledger_retire() { # + rm -f "${1}.since" "${1}".since.* 2>/dev/null + return 0 +} + +# Retire the away session's working records: the private digests, any leaked +# spool, any ledger temp, and any abandoned version-store staging directory +# (never a published version - those are retired only through +# away_ledger_version_retire / away_ledger_versions_retire_all). The text-free +# .status delivery evidence is deliberately kept, so accepted/failed/ +# unavailable outcomes stay auditable. +# +# , when 1, keeps the digest directory intact. Callers pass 1 +# when this call is reached on a continuation of an already-active away session +# (state/.afk was already present) rather than a genuinely fresh entry, so a +# crashed-daemon restart or a failed re-entry can never destroy digested items +# that Firstmate has not consumed or folded yet. +away_ledger_retire_working_records() { # [preserve-digests] + local state=$1 preserve=${2:-0} + [ "$preserve" -eq 1 ] || rm -rf "$(away_ledger_digest_dir "$state")" 2>/dev/null + rm -f "$(away_ledger_delivery_dir "$state")"/.spool.* \ + "$state"/.subsuper-escalations.since.* \ + "$state"/.subsuper-escalations.apply.* \ + "$state"/.subsuper-inject-wedged.apply.* \ + "$state"/.subsuper-inject-wedged.stage.* \ + "$state"/.subsuper-chat-delivery.apply.* \ + "$state"/.mint.* 2>/dev/null + rm -rf "$(away_ledger_versions_dir "$state")"/.staging.* \ + "$state"/.tg-away-digest.apply.* 2>/dev/null + return 0 +} + +# -------------------------------------------------------------------------- +# The immutable versioned batch store. +# +# The complete owned artifact unit for one away batch transaction is the +# escalation buffer, its ledger sidecar, the wedge marker, and the private +# digest directory - every artifact a crash or a failed re-entry must resume +# exactly once from. The text-free .status delivery evidence is NEVER part +# of this unit: it outlives every version of the batch above it, so +# accepted/failed/unavailable outcomes stay auditable regardless of how many +# batches come and go over it. +# +# A version is one COMPLETE copy of that unit in its own directory under +# state/tg-away-versions: +# +# .staging.XXXXXX/ a private in-progress build. Never listed, never read, +# never activated; swept as an away working record. +# v.-/ a published version. Built in staging, validated against +# its own manifest, and made visible by ONE rename, so a +# crash can only ever leave a staging directory behind, +# never a half-built version. The `.complete` marker +# written last inside staging is the readable proof of +# that, and any v.* directory lacking it is treated as +# incomplete: ignored by every reader and swept by +# away_ledger_version_gc. +# active the single owner-controlled pointer naming the version +# the live unit must hold. Written atomically, and that +# write IS the commit point of a switch. +# active.applied the version whose content has already been materialised +# onto the live paths. active != active.applied means a +# switch is still pending and is replayed, so an +# interrupted switch converges instead of leaving the live +# unit half-installed. +# .owner.lock the store's own owner lock. Publishing, switching, +# materialising, and retiring all run while exactly one +# process holds it, so the two away entry paths (the +# launcher and a direct daemon start, which share no other +# lock) can never interleave. +# +# Nothing is ever copied or merged INTO a published version, and no individual +# file is ever merged into the live unit: a switch installs one version's whole +# unit or leaves the live unit entirely alone, so a version's sidecar counts can +# never be paired with a foreign buffer. +# +# A published version is immutable in the strict sense: every reader of one goes +# through away_ledger_peek, which parses without ever writing, so not even the +# legacy-record-shape migration can touch it. That migration belongs to successor +# construction instead - away_ledger_version_publish normalises the record inside +# its private staging copy - so versions always carry a current-shape record and +# the live sidecar the copy came from is left exactly as the daemon owns it. +# +# EVERY ledger transition is a version transition, the daemon's own included: the +# escalation append, the phone reservation, the evidence-proven acceptance, the +# retired attempt and its retry schedule, the digest plus the accounted count it +# advances, the wedge evidence, and the in-session truncation all go through +# away_ledger_transact. It takes the owner lock, copies the live unit into private +# staging, applies exactly ONE change there, validates and publishes the complete +# successor, switches the active pointer, and only then rewrites the live paths +# from a version that is already durable. No transition writes a live artifact as +# its source of truth, so the live unit is always a projection of some committed +# version - never state invented outside one - and a crash leaves either the +# predecessor active or a committed successor whose projection is replayed. +# +# The superseded predecessor is retired as soon as its successor is applied, since +# the successor was built from it and holds everything it held. A retained version +# that is NOT the chain's predecessor - a lifecycle entry capture, a rollback whose +# switch was refused - is never touched by a transition and retires only once the +# return catch-up is acknowledged. +# +# A switch is only ever performed while no live away daemon holds the +# supervise-daemon lock - checked inside this owner (away_ledger_daemon_live) +# rather than trusted to callers, and reported as rc 3. The daemon is the sole +# writer of the live buffer and its sidecar with no shared lock around them, so +# installing a unit under a running daemon would overwrite a freshly appended +# escalation that had never been reserved, digested, or shown in captain chat. +# Callers therefore only ever switch BEFORE daemon start; a refused switch +# leaves every version untouched and the return catch-up folds them instead. +# +# Versions are read - never mutated - by away_ledger_fold_versions at return, +# and retired as WHOLE versions only once that catch-up is acknowledged. + +away_ledger_versions_dir() { # + printf '%s\n' "$1/tg-away-versions" +} + +# A published version name: minted by this owner, never derived from a live +# basename, and validated on every read so a pointer can only ever address a +# version directory inside the store. +away_ledger_version_name_valid() { # + case "${1:-}" in + v.*) ;; + *) return 1 ;; + esac + case "$1" in + *[!a-zA-Z0-9.-]*) return 1 ;; + esac + return 0 +} + +away_ledger_version_is_complete() { # + [ -n "${1:-}" ] && [ -d "$1" ] && [ -f "$1/.complete" ] && [ -f "$1/manifest" ] +} + +away_ledger_version_path() { # + away_ledger_version_name_valid "${2:-}" || return 1 + printf '%s\n' "$(away_ledger_versions_dir "$1")/$2" +} + +# Whether a live away daemon holds the supervise-daemon lock right now. Derived +# from alone - never from a caller-supplied claim - and fail-closed: an +# owner whose recorded pid is alive counts as live, so an ambiguous lock refuses +# a pointer switch rather than racing a possible writer. +away_ledger_daemon_live() { # + local lock="$1/.supervise-daemon.lock" owner pid + if [ -L "$lock" ]; then + owner=$(readlink "$lock" 2>/dev/null) || return 1 + [ -n "$owner" ] || return 1 + case "$owner" in + /*) ;; + *) owner="$1/$owner" ;; + esac + elif [ -d "$lock" ]; then + owner=$lock + else + return 1 + fi + pid=$(head -n 1 "$owner/pid" 2>/dev/null || true) + case "$pid" in + ''|*[!0-9]*) return 1 ;; + esac + kill -0 "$pid" 2>/dev/null +} + +# The store's own owner lock. Every mutation of the store - publishing a version, +# switching the active pointer, materialising it, retiring a version - happens +# while exactly one process holds it, so two away entries (the launcher and a +# direct daemon start, which do not share the launcher's lock) can never +# interleave a publication with a pointer switch. A lock whose recorded pid is +# gone is reclaimed, but only ever through the steal lock below, so two acquirers +# that both observed the same dead holder cannot both remove and both create - the +# race that would hand the store to two owners at once and let a publish interleave +# with a materialise. An incomplete lock is given a bounded grace first, so a +# racing acquirer's own half-written lock is never ripped out from under it. +away_ledger_lock_acquire() { # + local dir=$1 lock="$1/.owner.lock" attempt=0 incomplete=0 pid stale me + me=${BASHPID:-$$} + (umask 077; mkdir -p "$dir" 2>/dev/null) || return 1 + # Reentrant for the SAME process only, with a bounded depth. The daemon's + # TERM/INT cleanup runs its final flush inside this same shell, so without this + # a signal arriving mid-transaction would make every shutdown transition spin + # against a lock this process already holds - a stalled shutdown and a lost + # final flush. Exclusion against other processes is unchanged, and the recorded + # identity is re-verified against the lock on disk, so a lock lost or stolen in + # between is never treated as still held. + if [ "${AWAY_LEDGER_LOCK_DEPTH:-0}" -gt 0 ] \ + && [ "${AWAY_LEDGER_LOCK_DIR:-}" = "$dir" ] \ + && [ "${AWAY_LEDGER_LOCK_OWNER:-}" = "$me" ] \ + && [ "$(head -n 1 "$lock/pid" 2>/dev/null || true)" = "$me" ]; then + [ "$AWAY_LEDGER_LOCK_DEPTH" -lt 8 ] || return 1 + AWAY_LEDGER_LOCK_DEPTH=$((AWAY_LEDGER_LOCK_DEPTH + 1)) + return 0 + fi + while [ "$attempt" -lt 200 ]; do + attempt=$((attempt + 1)) + if mkdir "$lock" 2>/dev/null; then + if ! printf '%s\n' "$me" > "$lock/pid" 2>/dev/null; then + rm -rf -- "$lock" 2>/dev/null + return 1 + fi + AWAY_LEDGER_LOCK_DIR=$dir + AWAY_LEDGER_LOCK_OWNER=$me + AWAY_LEDGER_LOCK_DEPTH=1 + return 0 + fi + pid=$(head -n 1 "$lock/pid" 2>/dev/null || true) + stale=0 + case "$pid" in + ''|*[!0-9]*) + incomplete=$((incomplete + 1)) + if [ "$incomplete" -lt 20 ]; then + sleep 0.05 + continue + fi + stale=1 + ;; + *) + incomplete=0 + kill -0 "$pid" 2>/dev/null || stale=1 + ;; + esac + if [ "$stale" -eq 1 ]; then + _away_ledger_lock_steal "$lock" "$pid" + incomplete=0 + continue + fi + sleep 0.05 + done + return 1 +} + +# Reclaim a lock whose holder is provably gone, under a lock of its own. The +# holder identity is RE-READ while the steal lock is held: whoever loses the steal +# race then sees either no lock at all or a different (new) holder, and removes +# nothing. A steal lock whose own holder died is reclaimed the same way once, +# without recursing, so a reclaimer that died mid-steal cannot wedge the store. +_away_ledger_lock_steal() { # + local lock=$1 observed=$2 steal="$1.steal" holder current + if ! mkdir "$steal" 2>/dev/null; then + holder=$(head -n 1 "$steal/pid" 2>/dev/null || true) + case "$holder" in + ''|*[!0-9]*) rm -rf -- "$steal" 2>/dev/null ;; + *) kill -0 "$holder" 2>/dev/null || rm -rf -- "$steal" 2>/dev/null ;; + esac + return 0 + fi + printf '%s\n' "${BASHPID:-$$}" > "$steal/pid" 2>/dev/null || true + current=$(head -n 1 "$lock/pid" 2>/dev/null || true) + if [ "$current" = "$observed" ]; then + case "$current" in + ''|*[!0-9]*) rm -rf -- "$lock" 2>/dev/null ;; + *) kill -0 "$current" 2>/dev/null || rm -rf -- "$lock" 2>/dev/null ;; + esac + fi + rm -rf -- "$steal" 2>/dev/null + return 0 +} + +# Release one nesting level; the lock itself is only dropped by the outermost +# release, so a nested transition can never unlock the transaction containing it. +away_ledger_lock_release() { # + local dir=$1 lock="$1/.owner.lock" me + me=${BASHPID:-$$} + if [ "${AWAY_LEDGER_LOCK_DEPTH:-0}" -gt 0 ] && [ "${AWAY_LEDGER_LOCK_DIR:-}" = "$dir" ] \ + && [ "${AWAY_LEDGER_LOCK_OWNER:-}" = "$me" ]; then + AWAY_LEDGER_LOCK_DEPTH=$((AWAY_LEDGER_LOCK_DEPTH - 1)) + [ "$AWAY_LEDGER_LOCK_DEPTH" -eq 0 ] || return 0 + AWAY_LEDGER_LOCK_DIR='' + AWAY_LEDGER_LOCK_OWNER='' + fi + [ "$(head -n 1 "$lock/pid" 2>/dev/null || true)" = "$me" ] || return 0 + rm -rf -- "$lock" 2>/dev/null + return 0 +} + +# Same-directory mktemp plus mv, so a reader never sees a half-written pointer. +# The temp lives under the store's own .pointer. prefix, so it can never be +# mistaken for a version and is swept by away_ledger_version_gc. +away_ledger_pointer_write() { # active|active.applied + local dir=$1 base=$2 name=$3 tmp + away_ledger_version_name_valid "$name" || return 1 + case "$base" in + active|active.applied) ;; + *) return 1 ;; + esac + tmp=$(umask 077; mktemp "$dir/.pointer.XXXXXX" 2>/dev/null) || return 1 + if ! printf '%s\n' "$name" > "$tmp" 2>/dev/null || ! mv -f "$tmp" "$dir/$base" 2>/dev/null; then + rm -f -- "$tmp" 2>/dev/null + return 1 + fi + return 0 +} + +away_ledger_pointer_read() { # + local name + name=$(head -n 1 "${1:-}" 2>/dev/null || true) + away_ledger_version_name_valid "$name" || return 1 + printf '%s\n' "$name" +} + +# Publish the CURRENT live unit as one new immutable version and print its name. +# Built entirely inside a private staging directory, validated against the +# manifest it wrote, marked complete, and published by a single rename - so a +# failure or a crash at any point leaves only staging behind and never a +# partially published version a later caller could activate. +away_ledger_version_publish() { # + local state=$1 buf=$2 wedge=$3 dir rc + dir=$(away_ledger_versions_dir "$state") + fmx_private_artifact_dir_prepare "$dir" >/dev/null 2>&1 || return 1 + away_ledger_lock_acquire "$dir" || return 1 + _away_ledger_version_publish_locked "$state" "$buf" "$wedge" + rc=$? + away_ledger_lock_release "$dir" + return "$rc" +} + +_away_ledger_version_publish_locked() { # + local state=$1 buf=$2 wedge=$3 built staging + built=$(_away_ledger_stage_build "$state" "$buf" "$wedge") || return 1 + staging=${built#* } + [ -n "$staging" ] && [ -d "$staging" ] || return 1 + _away_ledger_stage_commit "$state" "$staging" +} + +# Copy the current live unit into a fresh private staging directory and print it. +# This is the ONLY reader of the live unit inside a transaction, so a successor +# version always starts from a complete, self-consistent copy of one unit rather +# than from artifacts gathered at different moments. +_away_ledger_stage_build() { # + local state=$1 buf=$2 wedge=$3 dir staging digest_dir slot artifact base + local -a artifacts slots + dir=$(away_ledger_versions_dir "$state") + staging=$(umask 077; mktemp -d "$dir/.staging.XXXXXX" 2>/dev/null) || return 1 + # Preferred base: the immutable active predecessor directory itself, so the + # successor is provably that version plus one transition - which is also what + # makes retiring the predecessor sound. Only used when the live unit is + # provably still that version's projection; a unit that diverged (an + # interrupted projection, or a writer outside this owner) is staged from live + # instead, so divergence is captured into the successor rather than discarded. + base=$(away_ledger_pointer_read "$dir/active" 2>/dev/null || true) + if [ -n "$base" ] \ + && [ "$base" = "$(away_ledger_pointer_read "$dir/active.applied" 2>/dev/null || true)" ] \ + && away_ledger_version_is_complete "$dir/$base" \ + && _away_ledger_unit_matches_version "$state" "$buf" "$wedge" "$dir/$base"; then + if ! cp -pR "$dir/$base/." "$staging/" 2>/dev/null \ + || ! rm -f -- "$staging/manifest" "$staging/.complete" 2>/dev/null; then + rm -rf -- "$staging" 2>/dev/null + return 1 + fi + printf '%s\t%s\n' "$base" "$staging" + return 0 + fi + artifacts=() + while IFS= read -r artifact; do artifacts+=("$artifact"); done \ + < <(_away_ledger_unit_paths "$state" "$buf" "$wedge") + slots=() + while IFS= read -r slot; do slots+=("$slot"); done < <(_away_ledger_unit_slots) + local i=0 + while [ "$i" -lt "${#slots[@]}" ]; do + artifact=${artifacts[$i]} + slot=${slots[$i]} + i=$((i + 1)) + [ -e "$artifact" ] || continue + if ! cp -p "$artifact" "$staging/$slot" 2>/dev/null; then + rm -rf -- "$staging" 2>/dev/null + return 1 + fi + done + digest_dir=$(away_ledger_digest_dir "$state") + if [ -d "$digest_dir" ]; then + if ! cp -pR "$digest_dir" "$staging/tg-away-digest" 2>/dev/null; then + rm -rf -- "$staging" 2>/dev/null + return 1 + fi + fi + printf '\t%s\n' "$staging" + return 0 +} + +# Whether the live unit is still exactly 's projection: every single +# file present-and-identical or absent on both sides, and the same digest item +# names. Anything else means something diverged after the projection. +_away_ledger_unit_matches_version() { # + local state=$1 buf=$2 wedge=$3 vdir=$4 slot artifact digest_dir i + local -a artifacts slots + artifacts=() + while IFS= read -r artifact; do artifacts+=("$artifact"); done \ + < <(_away_ledger_unit_paths "$state" "$buf" "$wedge") + slots=() + while IFS= read -r slot; do slots+=("$slot"); done < <(_away_ledger_unit_slots) + i=0 + while [ "$i" -lt "${#slots[@]}" ]; do + artifact=${artifacts[$i]} + slot=${slots[$i]} + i=$((i + 1)) + if [ -e "$vdir/$slot" ]; then + [ -f "$artifact" ] || return 1 + cmp -s "$vdir/$slot" "$artifact" || return 1 + else + [ ! -e "$artifact" ] || return 1 + fi + done + digest_dir=$(away_ledger_digest_dir "$state") + if [ -d "$vdir/tg-away-digest" ]; then + [ -d "$digest_dir" ] || return 1 + # shellcheck disable=SC2012 # Delivery evidence names are generated ids without spaces. + [ "$(cd "$vdir/tg-away-digest" && ls -1 2>/dev/null | sort)" \ + = "$(cd "$digest_dir" && ls -1 2>/dev/null | sort)" ] || return 1 + else + [ ! -d "$digest_dir" ] || return 1 + fi + return 0 +} + +# Validate a staged unit, describe it in its own manifest, mark it complete, and +# publish it by ONE rename - printing the published name. The manifest is written +# from the slots actually present, so it can never drift from what a mutator left +# behind, and the cross-pairing check refuses a record with no buffer to count. +# +# An earlier record shape is normalised HERE, in private staging: this is the only +# place that migration ever happens, so no published version and no live sidecar is +# ever rewritten by a read. A record that cannot be parsed at all is left exactly +# as found and published as-is: the version becomes the immutable quarantine of +# that malformed state, the fold over-reports from the whole buffer (never dropping +# an owed line), and away-mode entry is never wedged by unreadable predecessor +# bookkeeping. +_away_ledger_stage_commit() { # + local state=$1 staging=$2 dir slot name rec + local migrated id reserved confirmed accounted attempt retry did + dir=$(away_ledger_versions_dir "$state") + if [ -e "$staging/escalations.since" ] && [ ! -e "$staging/escalations" ]; then + rm -rf -- "$staging" 2>/dev/null + return 1 + fi + if [ -e "$staging/escalations.since" ]; then + rec=$(away_ledger_parse "$staging/escalations") + if [ "$rec" != unknown ]; then + IFS=' ' read -r migrated id reserved confirmed accounted attempt retry did <<< "$rec" + if [ "$migrated" -eq 1 ] \ + && ! _away_ledger_record_put "$staging/escalations" "$id" "$reserved" "$confirmed" \ + "$accounted" "$attempt" "$retry" "$did"; then + rm -rf -- "$staging" 2>/dev/null + return 1 + fi + fi + fi + if ! : > "$staging/manifest" 2>/dev/null; then + rm -rf -- "$staging" 2>/dev/null + return 1 + fi + for slot in escalations escalations.since wedge chat tg-away-digest; do + [ -e "$staging/$slot" ] || continue + if ! printf '%s\n' "$slot" >> "$staging/manifest" 2>/dev/null; then + rm -rf -- "$staging" 2>/dev/null + return 1 + fi + done + while IFS= read -r slot; do + [ -n "$slot" ] || continue + if [ ! -e "$staging/$slot" ]; then + rm -rf -- "$staging" 2>/dev/null + return 1 + fi + done < "$staging/manifest" + if ! : > "$staging/.complete" 2>/dev/null; then + rm -rf -- "$staging" 2>/dev/null + return 1 + fi + name="v.$(away_ledger_mint_id "$dir")" + # Never rename ONTO an existing entry: mv would move the staging directory + # inside it, leaving the predecessor published as though it were the successor. + if [ -e "$dir/$name" ] || ! mv "$staging" "$dir/$name" 2>/dev/null; then + rm -rf -- "$staging" 2>/dev/null + return 1 + fi + printf '%s\n' "$name" + return 0 +} + +# ONE ledger transition: under the owner lock, copy the live unit into private +# staging, let apply exactly one change to that copy, validate and +# publish it as a complete successor version, switch the active pointer atomically, +# and only then rewrite the live projection from the version that is already +# durable. A crash at any point leaves either the predecessor still active or a +# committed successor whose projection the next apply replays - never a live unit +# invented outside a version. +# +# The superseded predecessor is retired once the successor is applied: the +# successor was built from it and is a complete superset, so nothing that was +# evidence is discarded. Retained versions that are NOT this chain's predecessor - +# a lifecycle entry capture, a refused rollback - are never touched here and still +# retire only once the return catch-up is acknowledged. +away_ledger_transact() { # [args...] + local state=$1 buf=$2 wedge=$3 dir rc + shift 3 + dir=$(away_ledger_versions_dir "$state") + fmx_private_artifact_dir_prepare "$dir" >/dev/null 2>&1 || return 1 + away_ledger_lock_acquire "$dir" || return 1 + _away_ledger_transact_locked "$state" "$buf" "$wedge" "$@" + rc=$? + away_ledger_lock_release "$dir" + return "$rc" +} + +_away_ledger_transact_locked() { # [args...] + local state=$1 buf=$2 wedge=$3 dir staging name previous built + shift 3 + dir=$(away_ledger_versions_dir "$state") + built=$(_away_ledger_stage_build "$state" "$buf" "$wedge") || return 1 + previous=${built%% *} + staging=${built#* } + [ -n "$staging" ] && [ -d "$staging" ] || return 1 + if ! "$@" "$staging"; then + rm -rf -- "$staging" 2>/dev/null + return 1 + fi + name=$(_away_ledger_stage_commit "$state" "$staging") || return 1 + away_ledger_pointer_write "$dir" active "$name" || return 1 + away_ledger_version_materialise "$state" "$buf" "$wedge" "$dir/$name" || return 1 + away_ledger_pointer_write "$dir" active.applied "$name" || return 1 + # Only the version this successor was actually BUILT from is retired, so the + # superset claim is proven rather than assumed: a successor staged from a + # diverged live unit leaves its predecessor retained for the return fold, which + # already collapses identical lines. + if [ -n "$previous" ] && [ "$previous" != "$name" ] \ + && [ -d "$dir/$previous" ]; then + rm -rf -- "${dir:?}/${previous:?}" 2>/dev/null || true + fi + return 0 +} + +# Every COMPLETE published version, oldest first (the minted name carries the +# publication epoch). An incomplete directory is never listed, so no reader can +# act on a version that was never validated. +away_ledger_version_names() { # + local dir v name + dir=$(away_ledger_versions_dir "$1") + [ -d "$dir" ] || return 0 + for v in "$dir"/v.*; do + away_ledger_version_is_complete "$v" || continue + name=${v##*/} + away_ledger_version_name_valid "$name" || continue + printf '%s\n' "$name" + done | sort + return 0 +} + +# Drop everything the store must never act on: abandoned staging builds, leaked +# pointer temps, any version directory that is not provably complete, and any +# pointer left aiming at a version that is gone or was never completed. Never +# touches a complete version or a pointer that still resolves, so it is safe to run +# at any moment, including while a daemon is live. +# +# Clearing a dangling pointer is the only forward path: the content it named no +# longer exists, and leaving it would make every later apply - and so every away +# entry - refuse for good. Referenced evidence is preserved by construction, since +# a pointer is only ever cleared when the version it names is provably absent or +# incomplete. +away_ledger_version_gc() { # + local dir v pointer name result=0 + dir=$(away_ledger_versions_dir "$1") + [ -d "$dir" ] || return 0 + away_ledger_lock_acquire "$dir" || return 1 + rm -rf -- "$dir"/.staging.* 2>/dev/null + rm -f -- "$dir"/.pointer.* "$dir"/.mint.* 2>/dev/null + for v in "$dir"/v.*; do + [ -e "$v" ] || continue + away_ledger_version_is_complete "$v" && continue + rm -rf -- "$v" 2>/dev/null || result=1 + done + for pointer in active active.applied; do + [ -e "$dir/$pointer" ] || continue + name=$(away_ledger_pointer_read "$dir/$pointer" 2>/dev/null || true) + if [ -n "$name" ] && away_ledger_version_is_complete "$dir/$name"; then + continue + fi + rm -f -- "$dir/$pointer" 2>/dev/null || result=1 + done + away_ledger_lock_release "$dir" + return "$result" +} + +# THE pointer switch, and the only way a version's content ever reaches the live +# paths. Refuses an unknown or incomplete version (rc 1) and refuses outright +# while a live daemon holds the lock (rc 3); either way every version and the +# live unit are left exactly as they were. +away_ledger_version_activate() { # + local state=$1 buf=$2 wedge=$3 name=$4 dir vdir rc + vdir=$(away_ledger_version_path "$state" "$name") || return 1 + away_ledger_version_is_complete "$vdir" || return 1 + if away_ledger_daemon_live "$state"; then + return 3 + fi + dir=$(away_ledger_versions_dir "$state") + away_ledger_lock_acquire "$dir" || return 1 + if away_ledger_pointer_write "$dir" active "$name"; then + _away_ledger_version_apply_locked "$state" "$buf" "$wedge" + rc=$? + else + rc=1 + fi + away_ledger_lock_release "$dir" + return "$rc" +} + +# Replay a switch whose materialisation has not completed. Idempotent: with no +# pointer, or with the pointer already applied, it is a no-op. Callers run it +# before daemon start so an interrupted switch converges; it refuses (rc 3) +# while a daemon is live, so it can never race the daemon's own writes. +away_ledger_version_apply_pending() { # + local state=$1 buf=$2 wedge=$3 dir rc + dir=$(away_ledger_versions_dir "$state") + [ -d "$dir" ] || return 0 + away_ledger_lock_acquire "$dir" || return 1 + _away_ledger_version_apply_locked "$state" "$buf" "$wedge" + rc=$? + away_ledger_lock_release "$dir" + return "$rc" +} + +_away_ledger_version_apply_locked() { # + local state=$1 buf=$2 wedge=$3 dir name applied vdir + dir=$(away_ledger_versions_dir "$state") + [ -d "$dir" ] || return 0 + name=$(away_ledger_pointer_read "$dir/active") || return 0 + applied=$(away_ledger_pointer_read "$dir/active.applied" 2>/dev/null || true) + [ "$name" != "$applied" ] || return 0 + vdir="$dir/$name" + away_ledger_version_is_complete "$vdir" || return 1 + if away_ledger_daemon_live "$state"; then + return 3 + fi + away_ledger_version_materialise "$state" "$buf" "$wedge" "$vdir" || return 1 + away_ledger_pointer_write "$dir" active.applied "$name" +} + +# Install ONE version's whole unit onto the live paths, dropping whatever is +# there first, so no live artifact is ever left paired with a foreign version's +# counts. Every file lands through a same-directory mktemp-then-mv and the +# digest directory through a staged rename, so a reader elsewhere can never +# observe a half-copied artifact. A failure leaves the pointer unapplied, which +# is what makes the replay above safe to re-run. +away_ledger_version_materialise() { # + local state=$1 buf=$2 wedge=$3 vdir=$4 digest_dir slot artifact tmp i + local -a artifacts slots + artifacts=() + while IFS= read -r artifact; do artifacts+=("$artifact"); done \ + < <(_away_ledger_unit_paths "$state" "$buf" "$wedge") + slots=() + while IFS= read -r slot; do slots+=("$slot"); done < <(_away_ledger_unit_slots) + digest_dir=$(away_ledger_digest_dir "$state") + rm -f -- "${artifacts[@]}" 2>/dev/null || return 1 + rm -rf -- "$digest_dir" 2>/dev/null || return 1 + i=0 + while [ "$i" -lt "${#slots[@]}" ]; do + artifact=${artifacts[$i]} + slot=${slots[$i]} + i=$((i + 1)) + [ -e "$vdir/$slot" ] || continue + tmp=$(umask 077; mktemp "${artifact}.apply.XXXXXX" 2>/dev/null) || return 1 + if ! cp -p "$vdir/$slot" "$tmp" 2>/dev/null || ! mv -f "$tmp" "$artifact" 2>/dev/null; then + rm -f -- "$tmp" 2>/dev/null + return 1 + fi + done + if [ -d "$vdir/tg-away-digest" ]; then + tmp=$(umask 077; mktemp -d "$state/.tg-away-digest.apply.XXXXXX" 2>/dev/null) || return 1 + if ! cp -pR "$vdir/tg-away-digest/." "$tmp/" 2>/dev/null \ + || ! chmod 700 "$tmp" 2>/dev/null \ + || ! mv "$tmp" "$digest_dir" 2>/dev/null; then + rm -rf -- "$tmp" 2>/dev/null + return 1 + fi + fi + return 0 +} + +# Retire ONE whole version. Any pointer naming it is cleared FIRST, so a crash +# can never leave a pointer aimed at a version that no longer exists. +away_ledger_version_retire() { # + local state=$1 name=$2 dir vdir pointer current result=0 + vdir=$(away_ledger_version_path "$state" "$name") || return 1 + dir=$(away_ledger_versions_dir "$state") + [ -d "$dir" ] || return 0 + away_ledger_lock_acquire "$dir" || return 1 + for pointer in active active.applied; do + current=$(away_ledger_pointer_read "$dir/$pointer" 2>/dev/null || true) + [ "$current" = "$name" ] || continue + rm -f -- "$dir/$pointer" 2>/dev/null || result=1 + done + rm -rf -- "$vdir" 2>/dev/null || result=1 + away_ledger_lock_release "$dir" + return "$result" +} + +# Retire the WHOLE version store. Called only once a return catch-up has been +# acknowledged: every version's actionable content has reached that catch-up's +# durable evidence by then, so nothing retired here can resurface as a duplicate +# delivery, and nothing is retired while a catch-up is still pending. +away_ledger_versions_retire_all() { # + local dir name result=0 + dir=$(away_ledger_versions_dir "$1") + [ -d "$dir" ] || return 0 + away_ledger_lock_acquire "$dir" || return 1 + rm -f -- "$dir/active" "$dir/active.applied" "$dir"/.pointer.* "$dir"/.mint.* 2>/dev/null \ + || result=1 + rm -rf -- "$dir"/v.* "$dir"/.staging.* 2>/dev/null || result=1 + # Removal is only acknowledged when the store is provably empty of versions and + # pointers: a caller that keeps a gate open on this status must not be told the + # catch-up is settled while a version survives to be folded again. + while IFS= read -r name; do + [ -z "$name" ] || result=1 + done </dev/null || true + return "$result" +} + +# The ONE entry-boundary transaction every away entry goes through - the launcher +# and a direct daemon start alike - printing the name of the version it captured. +# Sweep what was never published, replay an interrupted pointer switch (the only +# moment a version's content may reach the live paths, and refused by the owner +# while a daemon is live), then capture the predecessor unit as one complete +# immutable version. +# +# A caller retires the live unit ONLY after this has succeeded, so no entry path +# can delete a crashed session's un-flushed escalation lines with no surviving +# copy - the evidence-loss class this store exists to close. Because the capture +# lives here rather than in one caller, a future entry path cannot skip it by +# omission. +away_ledger_entry_capture() { # + local state=$1 buf=$2 wedge=$3 rc=0 + away_ledger_version_gc "$state" || true + away_ledger_version_apply_pending "$state" "$buf" "$wedge" || rc=$? + [ "$rc" -ne 1 ] || return 1 + away_ledger_version_publish "$state" "$buf" "$wedge" +} + +# Print the lines of that no digest has accounted for yet, one per line - +# the lines that still have to reach captain chat. Falls back to the whole +# buffer whenever the record is absent or `unknown`, so an unreadable ledger +# over-reports to the visible catch-up instead of dropping a captain-relevant +# line. +# +# Reads through away_ledger_peek, never away_ledger_read: this runs against +# published immutable versions as well as the live buffer, and a migrating read +# would both break that immutability and - if the migration write failed - report +# `unknown` and re-offer lines the version's own digests already carry. +away_ledger_unaccounted_lines() { # + local buf=$1 rec accounted=0 n + [ -s "$buf" ] || return 0 + if [ -f "${buf}.since" ]; then + rec=$(away_ledger_peek "$buf" 2>/dev/null || true) + if [ -n "$rec" ] && [ "$rec" != unknown ]; then + IFS=' ' read -r _ _ _ accounted _ _ _ <<< "$rec" + away_ledger_is_count "$accounted" || accounted=0 + fi + fi + n=$(( $(wc -l < "$buf" 2>/dev/null || echo 0) )) + [ "$n" -gt "$accounted" ] || return 0 + tail -n "+$((accounted + 1))" "$buf" 2>/dev/null || return 1 + return 0 +} + +# Print ONE kind of actionable content held by every RETAINED version, so a +# version whose switch was refused (a live daemon, a failed stop), whose +# transaction was abandoned, or whose content a fresh away entry retired from the +# live paths still reaches the return catch-up instead of ageing out silently. +# +# Read-only: nothing is retired, renamed, or marked here, so a crash mid +# catch-up simply re-reads the same versions next time, and the retirement that +# eventually removes them happens only after acknowledgement. +# +# One kind per call, so a caller can append each to the SAME evidence kind its +# own live read uses. That is what makes each logical escalation appear exactly +# once: a version's content is a prefix-subset of the live unit whenever the live +# unit was materialised from it, and identical lines under one evidence kind +# collapse in the caller's own dedupe instead of being presented twice under two +# different labels. +# +# escalations the lines past each version's OWN accounted count - never +# anything that version's own digests already carry. +# wedges each version's wedge marker line. +# digests each version's digest items whose delivery id is not already +# live, since one delivery id always names the same text. +away_ledger_fold_versions() { # escalations|wedges|digests + local state=$1 kind=$2 dir name vdir digest_dir f base + case "$kind" in + escalations|wedges|digests) ;; + *) return 1 ;; + esac + dir=$(away_ledger_versions_dir "$state") + [ -d "$dir" ] || return 0 + digest_dir=$(away_ledger_digest_dir "$state") + while IFS= read -r name; do + [ -n "$name" ] || continue + vdir="$dir/$name" + case "$kind" in + escalations) + away_ledger_unaccounted_lines "$vdir/escalations" || return 1 + ;; + wedges) + if [ -s "$vdir/wedge" ]; then + head -n 1 "$vdir/wedge" 2>/dev/null || return 1 + fi + ;; + digests) + for f in "$vdir"/tg-away-digest/*.items; do + [ -f "$f" ] || continue + base=${f##*/} + if [ -e "$digest_dir/$base" ]; then + continue + fi + cat "$f" 2>/dev/null || return 1 + done + ;; + esac + done <, when 1, keeps the digest directory intact (see +# away_ledger_retire_working_records) - callers pass 1 on a continuation of an +# already-active away session rather than a genuinely fresh entry. +away_ledger_retire_batch() { # [preserve-digests] + local state=$1 buf=$2 wedge=$3 preserve=${4:-0} dir result=0 + rm -f "$buf" "$wedge" "$(away_ledger_chat_of "$state")" 2>/dev/null || result=1 + away_ledger_retire "$buf" || result=1 + away_ledger_retire_working_records "$state" "$preserve" || result=1 + # The retired unit is no longer any version's projection, so the chain head is + # cleared too: the next transition stages a fresh unit instead of resurrecting a + # retired batch from a stale active pointer. The version DIRECTORIES stay - they + # are the evidence the return catch-up folds and only acknowledgement retires. + dir=$(away_ledger_versions_dir "$state") + if [ -d "$dir" ]; then + if away_ledger_lock_acquire "$dir"; then + rm -f -- "$dir/active" "$dir/active.applied" 2>/dev/null || result=1 + away_ledger_lock_release "$dir" + else + result=1 + fi + fi + return "$result" +} diff --git a/bin/fm-supervise-daemon.sh b/bin/fm-supervise-daemon.sh index 6ec14aed4b..c5c0813999 100755 --- a/bin/fm-supervise-daemon.sh +++ b/bin/fm-supervise-daemon.sh @@ -122,7 +122,17 @@ # FM_WEDGE_ALARM_TIMEOUT_SECS seconds allowed for each notifier before # its watchdog terminates it and continues to the # next channel (default 10; invalid/zero uses the -# default). +# default). Also bounds the away-mode Telegram +# escalation send (telegram_away_send_file). +# FM_TG_AWAY_RETRY_SECS seconds a retired proven-local away Telegram +# attempt waits before the same lines may be +# offered to the phone again; honored only when +# positive, else the batch window, else the batch +# default (telegram_away_retry_secs; never 0). +# FM_TG_AWAY_EXEC away Telegram seam: "discard" sends nothing and +# reports the batch as off. Unset in production. +# When SOURCED the daemon defaults this to +# "discard" so no test reaches the real phone. # FM_INJECT_CONFIRM_RETRIES Enter-retry attempts on a swallowed Enter # (default 3); the digest is typed once, only # Enter is retried. Composer-empty detection is @@ -160,6 +170,23 @@ FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" # shellcheck source=bin/fm-operational-input.sh . "$FM_DAEMON_DIR/fm-operational-input.sh" +# Private-artifact identity helpers (fmx_private_artifact_*) used by the away +# delivery records below. Sourced directly rather than relying on fm-tg-lib.sh's +# transitive source, so the dependency is explicit at this call site. +# shellcheck source=bin/fm-x-lib.sh +. "$FM_DAEMON_DIR/fm-x-lib.sh" + +# The single owner of the away batch ledger (identity, phone reservation, +# confirmed acceptance, digest retention, retirement). The daemon, away start, and +# away return all transition this one owner instead of keeping local counters. +# shellcheck source=bin/fm-away-ledger-lib.sh +. "$FM_DAEMON_DIR/fm-away-ledger-lib.sh" + +# Telegram mode's existing authenticated outbound transport owner. +# The daemon uses only its stdin sender helper and never reads credentials. +# shellcheck source=bin/fm-tg-lib.sh +. "$FM_DAEMON_DIR/fm-tg-lib.sh" + # Shared wake classifier (last_status_line, status_is_captain_relevant, # window_to_task, scan_captain_relevant_statuses). The SAME library backs the # always-on watcher's triage, so the captain-relevant verb set and the @@ -234,11 +261,6 @@ _file_age() { # seconds since mtime; very large if missing echo $(( $(_now) - m )) } -_hash_text() { - if command -v md5 >/dev/null 2>&1; then printf '%s' "$1" | md5 -q - else printf '%s' "$1" | md5sum | cut -d ' ' -f1; fi -} - # --- presence-gating helpers (PURE-ish: side-effect-free reads of state) ----- # afk_active: 0 if the durable away-mode flag exists, 1 otherwise. afk_active() { # @@ -423,6 +445,19 @@ classify_unknown() { # # Buffer: state/.subsuper-escalations one distilled line per escalation. # Seen: state/.subsuper-seen-status- last status line the scan # escalated, so the catch-all does not re-fire the same terminal. +# Away: state/tg-away-delivery/.status text-free delivery evidence. +# state/.subsuper-escalations.since the batch ledger (identity, +# reserved/confirmed/accounted counts, attempt ordinal, retry schedule) +# owned by fm-away-ledger-lib.sh. +# state/tg-away-digest/.items private 0600 copy of the +# accepted items, the working record the receipt points Firstmate at. +# state/.subsuper-chat-delivery the batch's in-session (captain +# chat) delivery state, so a bookkeeping failure after a confirmed +# submit can never become a second delivery of the same batch. +# state/tg-away-versions/v./ one complete immutable copy +# of that whole unit per ledger transition, published under the store's +# owner lock; every live artifact above is the projection of whichever +# version the store's single active pointer names. _stale_key() { printf '%s' "$1" | tr ':/.' '___'; } @@ -610,27 +645,420 @@ stale_window_is_busy() { # | fm_busy_lines_match "$harness" } +# --- away-mode escalation batch (ledger-governed) ---------------------------- +# bin/fm-away-ledger-lib.sh is the single owner of the batch ledger: the per-line +# phone reservation, the evidence-proven confirmed count, the accounted +# (digest-recorded) count, the private digest, and the retirement of both. Nothing +# below keeps its own counters - every decision here is a query or a transition on +# that owner, and an `unknown` ledger fails CLOSED: nothing reaches the phone and +# the whole buffer goes to the visible in-session escalation. escalate_add() { # local state=$1 item=$2 buf buf="$state/.subsuper-escalations" - [ -s "$buf" ] || _now > "${buf}.since" - printf '%s\n' "$item" >> "$buf" + if [ ! -s "$buf" ]; then + away_ledger_open "$buf" || away_ledger_retire "$buf" + fi + if ! away_ledger_append "$buf" "$item"; then + log "ERROR: away batch append refused; the escalation is NOT buffered (ledger owner could not commit a successor version). The heartbeat scan and wake queue re-derive it." + return 1 + fi + return 0 +} + +# Send one batched away alert through Telegram mode's existing phone-inbox tg +# client. The durable record stores only a delivery id, result class, and epoch. +# It never stores alert text, credentials, chat ids, or raw sender diagnostics. +# +# is the ledger's reserved count, so the delivery id names an exact +# (batch, first-unreserved-line) pair and a batch that grew while the in-session +# receipt was wedged sends only its NEW lines. +# +# Every argument is required and individually validated before any sender or ledger +# mutation, and the ledger must already describe this exact batch with +# `offset == reserved == confirmed` and `total > offset`; anything else refuses to +# send, so no caller can reach the phone without exactly-once accounting and no +# delivery id is minted outside the ledger owner. +# +# Ordering, all of it before the network call: the ledger reserves every line +# about to be sent, then an attempting evidence record is written. Either write +# failing aborts the send outright, so an unrecordable delivery can never become a +# duplicate phone alert. Only an accepted evidence record transitions the ledger's +# confirmed count, over a contiguous prefix only. +# +# The transport outcome is classified into exactly two non-accepted proof classes, +# never inferred from a bare non-zero status: +# proven-local nothing reached the network - no runnable client +# (fmtg_client_runnable, checked before the watchdog ever runs), +# or the watchdog's own pre-launch guard (125), which +# telegram_away_send_file makes unforgeable by remapping any +# 124/125 the client itself might return to a generic ambiguous +# status, so 125 here can only be the watchdog's sentinel. The +# attempt is retired through the ledger: its claim comes back AND +# the batch's attempt ordinal advances, so the same lines retry +# under a fresh delivery id while this attempt's evidence stays +# auditable. +# ambiguous the request may have reached Telegram - a watchdog timeout, or a +# client that ran and exited non-zero (including a disappeared +# client racing the runnable check, or 124/125 from the client +# itself), which cannot be distinguished from a lost response to a +# send Telegram accepted. The claim is KEPT, the ordinal does not +# move, so those lines are never offered to the phone again and +# the batch falls back visibly in session instead. +# +# The send itself runs under the same bounded watchdog every other external +# notifier in this file uses (FM_WEDGE_ALARM_TIMEOUT_SECS), so a hung phone client +# can never stall housekeeping or the ~1s shutdown path. +telegram_away_send_file() { # + local rc + fmtg_send_stdin < "$1" + rc=$? + case "$rc" in + 124|125) return 1 ;; + *) return "$rc" ;; + esac +} + +# How long a retired proven-local attempt waits before the same lines may be +# offered to the phone again. ALWAYS positive: FM_TG_AWAY_RETRY_SECS wins only when +# it supplies a positive bound of its own, otherwise the batch window is used and a +# non-positive or malformed window falls back to the default. Immediate batching +# (FM_ESCALATE_BATCH_SECS=0) therefore keeps flushing every tick as documented +# while a persistent local failure still re-attempts on a bounded cadence instead +# of once per tick. +telegram_away_retry_secs() { + local secs + for secs in "${FM_TG_AWAY_RETRY_SECS:-}" "${FM_ESCALATE_BATCH_SECS:-}"; do + if away_ledger_is_count "$secs" && [ "$secs" -gt 0 ]; then + printf '%s\n' "$secs" + return 0 + fi + done + printf '%s\n' "$ESCALATE_BATCH_SECS_DEFAULT" } -# Flush the escalation buffer as ONE batched, single-line digest to the -# supervisor pane. Returns 0 on successful inject (or empty buffer), non-zero on -# inject failure (buffer preserved for retry / catch-up). +telegram_away_deliver() { # + local state=$1 body=$2 offset=${3:-} total=${4:-} accounted=${5:-} batch_id=${6:-} + local buf rec led_id led_reserved led_confirmed led_accounted led_attempt led_retry led_did + local delivery_id dir base record status now rc alert spool + if [ "${FM_TG_AWAY_EXEC:-live}" = discard ] || ! afk_active "$state" || ! fmtg_enabled "$FM_HOME"; then + printf 'off|none\n' + return 0 + fi + buf="$state/.subsuper-escalations" + away_ledger_is_count "$offset" "$total" "$accounted" \ + || { printf 'unavailable|none\n'; return 0; } + rec=$(away_ledger_read "$buf") + [ "$rec" != unknown ] || { printf 'unavailable|none\n'; return 0; } + IFS=' ' read -r led_id led_reserved led_confirmed led_accounted led_attempt led_retry led_did <<< "$rec" + if ! away_ledger_retry_due "$led_retry"; then + printf 'deferred|%s\n' "$led_did" + return 0 + fi + if [ "$batch_id" != "$led_id" ] \ + || [ "$offset" -ne "$led_reserved" ] || [ "$offset" -ne "$led_confirmed" ] \ + || [ "$accounted" -ne "$led_accounted" ] || [ "$total" -le "$offset" ]; then + printf 'unavailable|none\n' + return 0 + fi + delivery_id=$(away_ledger_delivery_id "$batch_id" "$led_attempt" "$offset" "$body") \ + || { printf 'unavailable|none\n'; return 0; } + dir=$(away_ledger_delivery_dir "$state") + base="$delivery_id.status" + if fmx_private_artifact_file_valid "$dir" "$base" 600; then + record=$(cat "$dir/$base" 2>/dev/null || true) + status=${record%% *} + case "$status" in + accepted) + telegram_away_settle_accepted "$state" "$buf" "$delivery_id" "$offset" "$total" "$accounted" + return 0 + ;; + failed|unavailable) + printf '%s|%s\n' "$status" "$delivery_id" + return 0 + ;; + attempting) + now=$(_now) + printf 'failed %s uncertain_previous_attempt\n' "$now" \ + | fmx_private_artifact_publish_stdin "$dir" "$base" 600 2>/dev/null || true + printf 'uncertain|%s\n' "$delivery_id" + return 0 + ;; + esac + elif [ -e "$dir/$base" ] || [ -L "$dir/$base" ]; then + printf 'unavailable|%s\n' "$delivery_id" + return 0 + fi + + # No durable ownership, no attempt: publishing an outcome here would tombstone a + # delivery id that never reserved or sent anything, and the ledger write that + # would retire it is the very write that just failed. Leaving the id untouched + # lets the next flush retry cleanly once the state dir is writable again, while + # this flush still reports the outcome in session. + if ! away_ledger_reserve "$buf" "$total" "$delivery_id"; then + printf 'unavailable|none\n' + return 0 + fi + + now=$(_now) + if ! printf 'attempting %s\n' "$now" \ + | fmx_private_artifact_publish_stdin_once "$dir" "$base" 600 2>/dev/null; then + away_ledger_release "$buf" "$offset" "$delivery_id" "$(telegram_away_retry_secs)" + printf 'unavailable|%s\n' "$delivery_id" + return 0 + fi + + if ! fmtg_client_runnable; then + away_ledger_release "$buf" "$offset" "$delivery_id" "$(telegram_away_retry_secs)" + printf 'unavailable %s sender_missing\n' "$now" \ + | fmx_private_artifact_publish_stdin "$dir" "$base" 600 2>/dev/null || true + printf 'unavailable|%s\n' "$delivery_id" + return 0 + fi + + alert=$(printf 'Firstmate needs you: %s Notice only. This message does not approve a merge, privileged change, destructive action, or security-sensitive action.' "$body") + # The watchdog backgrounds its command, and a backgrounded command's stdin is + # /dev/null, so the text travels via the ledger owner's private spool file, + # which the sender redirects from itself - never through a command argument. + if ! spool=$(away_ledger_spool_create "$state") \ + || ! printf '%s' "$alert" > "$spool" 2>/dev/null; then + rm -f -- "${spool:-}" 2>/dev/null || true + away_ledger_release "$buf" "$offset" "$delivery_id" "$(telegram_away_retry_secs)" + now=$(_now) + printf 'failed %s sender_spool_unavailable\n' "$now" \ + | fmx_private_artifact_publish_stdin "$dir" "$base" 600 2>/dev/null || true + printf 'failed|%s\n' "$delivery_id" + return 0 + fi + if wedge_alarm_run_bounded telegram telegram_away_send_file "$spool" \ + >/dev/null 2>/dev/null; then + rc=0 + else + rc=$? + fi + rm -f -- "$spool" + now=$(_now) + case "$rc" in + 0) + printf 'accepted %s\n' "$now" \ + | fmx_private_artifact_publish_stdin "$dir" "$base" 600 2>/dev/null || true + telegram_away_settle_accepted "$state" "$buf" "$delivery_id" "$offset" "$total" "$accounted" + ;; + 125) # proven local: the watchdog never started the client + away_ledger_release "$buf" "$offset" "$delivery_id" "$(telegram_away_retry_secs)" + printf 'failed %s sender_watchdog_unavailable\n' "$now" \ + | fmx_private_artifact_publish_stdin "$dir" "$base" 600 2>/dev/null || true + printf 'failed|%s\n' "$delivery_id" + ;; + 124) # ambiguous: the request was in flight when the watchdog fired + printf 'failed %s sender_timeout\n' "$now" \ + | fmx_private_artifact_publish_stdin "$dir" "$base" 600 2>/dev/null || true + printf 'uncertain|%s\n' "$delivery_id" + ;; + *) # ambiguous: the client ran and returned non-zero, which proves nothing + printf 'failed %s sender_uncertain_response\n' "$now" \ + | fmx_private_artifact_publish_stdin "$dir" "$base" 600 2>/dev/null || true + printf 'uncertain|%s\n' "$delivery_id" + ;; + esac + return 0 +} + +# Turn an accepted evidence record into ledger state: confirm exactly the lines +# this delivery carried - a contiguous extension of the proven prefix, refused +# otherwise - then record the digest from the accounted position up to the newly +# confirmed count. `accepted` means Firstmate can reconcile from the digests alone; +# `accepted_undigested` means the caller must carry the still-unrecorded items in +# the in-session message instead. +telegram_away_settle_accepted() { # + local state=$1 buf=$2 delivery_id=$3 offset=$4 total=$5 accounted=$6 + if ! away_ledger_confirm "$buf" "$offset" "$total" "$delivery_id"; then + printf 'accepted_undigested|%s\n' "$delivery_id" + return 0 + fi + if away_ledger_digest_record "$state" "$buf" "$delivery_id" "$accounted" "$total"; then + printf 'accepted|%s\n' "$delivery_id" + else + printf 'accepted_undigested|%s\n' "$delivery_id" + fi + return 0 +} + +# Reconcile a reservation that outran the confirmed count against the durable +# delivery evidence, so a restart never has to GUESS whether the phone got it: +# no evidence at all -> the crash preceded the send, release the reservation and +# let the lines be offered to the phone again +# attempting -> outcome unknowable, retire the attempt and stay uncertain +# accepted -> confirm the reservation (contiguous: a send only ever +# starts at the confirmed count) +# failed/unavailable -> a kept reservation here means a maybe-delivered send +# (timeout, or a client that ran and returned non-zero), so +# stay uncertain rather than claim either way +telegram_away_recover() { # + local state=$1 buf=$2 reserved=$3 confirmed=$4 did=$5 dir base record status now + [ "$reserved" -gt "$confirmed" ] || { printf 'settled\n'; return 0; } + dir=$(away_ledger_delivery_dir "$state") + base="$did.status" + if [ "$did" = none ]; then + away_ledger_release "$buf" "$confirmed" "$did" || true + printf 'released\n' + return 0 + fi + if ! fmx_private_artifact_file_valid "$dir" "$base" 600; then + if [ -e "$dir/$base" ] || [ -L "$dir/$base" ]; then printf 'uncertain\n'; return 0; fi + away_ledger_release "$buf" "$confirmed" "$did" || true + printf 'released\n' + return 0 + fi + record=$(cat "$dir/$base" 2>/dev/null || true) + status=${record%% *} + case "$status" in + accepted) + away_ledger_confirm "$buf" "$confirmed" "$reserved" "$did" \ + || { printf 'uncertain\n'; return 0; } + printf 'confirmed\n' + ;; + attempting) + now=$(_now) + printf 'failed %s uncertain_previous_attempt\n' "$now" \ + | fmx_private_artifact_publish_stdin "$dir" "$base" 600 2>/dev/null || true + printf 'uncertain\n' + ;; + *) + printf 'uncertain\n' + ;; + esac + return 0 +} + +# Flush the escalation buffer as one batched digest, driven entirely by the ledger: +# lines past `reserved` -> the only lines the phone may ever be offered, and +# only while no earlier reservation is still unproven +# (`confirmed == reserved`), so an uncertain send stops +# this batch from reaching the phone again at all +# lines past `accounted` -> the only lines captain chat still needs to see +# A fully accepted and recorded delivery injects just a private receipt naming the +# batch's digests, so the alert contents are never repeated in captain chat. Every +# other outcome - uncertain, failed, unavailable, unknown ledger, or Telegram mode +# off - carries the not-yet-recorded items into the visible in-session escalation +# and labels the Telegram outcome honestly. The buffer clears only after the +# in-session submit is confirmed. escalate_flush() { # - local state=$1 buf item n msg + local state=$1 buf n body msg rec recovery tg_result tg_status tg_id + local batch_id reserved confirmed accounted attempt retry did pending_body chat_body chat_pending + local chat_delivered chat_start + local prior_note pointer buf="$state/.subsuper-escalations" [ -s "$buf" ] || return 0 - n=$(wc -l < "$buf" 2>/dev/null || echo 0) - # Join buffered items with the literal " | " separator into one digest line. - msg=$(awk 'NR>1{printf " | "} {printf "%s",$0} END{print ""}' "$buf" 2>/dev/null) - # Single-line wrapper: no embedded newlines (inject_msg also collapses as a - # safety net, but keeping the source single-line makes the intent explicit). - msg=$(printf 'Supervisor escalate (%s event(s)): %s (pre-read; re-arm not needed — watcher daemon-managed)' "$n" "$msg") - if inject_msg "$msg" "$state"; then : > "$buf"; rm -f "${buf}.since" "$state/.subsuper-inject-wedged"; return 0; fi + n=$(( $(wc -l < "$buf" 2>/dev/null || echo 0) )) + body=$(awk 'NR>1{printf " | "} {printf "%s",$0} END{print ""}' "$buf" 2>/dev/null) + batch_id=none; reserved=0; confirmed=0; accounted=0; attempt=0; retry=0; did=none + tg_status=''; tg_id=none + rec=$(away_ledger_read "$buf") + if [ "$rec" != unknown ]; then + IFS=' ' read -r batch_id reserved confirmed accounted attempt retry did <<< "$rec" + # A batch only ever grows, so a reservation past its line total means the + # ledger does not describe this buffer. + [ "$reserved" -le "$n" ] || rec=unknown + fi + # Captain chat already provably received this whole batch and only the + # bookkeeping that should have followed is outstanding. Retry that and nothing + # else: re-injecting would duplicate an escalation the captain has. + chat_delivered=$(away_ledger_chat_delivered "$state" "$batch_id") + if [ "$chat_delivered" -ge "$n" ]; then + away_ledger_truncate "$buf" || return 1 + return 0 + fi + if [ "$rec" != unknown ]; then + recovery=$(telegram_away_recover "$state" "$buf" "$reserved" "$confirmed" "$did") + if [ "$recovery" != settled ]; then + rec=$(away_ledger_read "$buf") + [ "$rec" = unknown ] \ + || IFS=' ' read -r batch_id reserved confirmed accounted attempt retry did <<< "$rec" + fi + fi + if [ "$rec" = unknown ]; then + tg_status=unknown; batch_id=none; reserved=0; confirmed=0; accounted=0; attempt=0; retry=0; did=none + elif [ "$n" -gt "$reserved" ] && [ "$confirmed" -eq "$reserved" ]; then + pending_body=$(tail -n "+$((reserved + 1))" "$buf" 2>/dev/null \ + | awk 'NR>1{printf " | "} {printf "%s",$0} END{print ""}') + tg_result=$(telegram_away_deliver "$state" "$pending_body" "$reserved" "$n" "$accounted" "$batch_id") + tg_status=${tg_result%%|*} + tg_id=${tg_result#*|} + rec=$(away_ledger_read "$buf") + [ "$rec" = unknown ] \ + || IFS=' ' read -r batch_id reserved confirmed accounted attempt retry did <<< "$rec" + elif [ "$confirmed" -ge "$n" ]; then + tg_id=$did + tg_status=accepted + [ "$accounted" -ge "$n" ] || tg_status=accepted_undigested + else + tg_id=$did + tg_status=uncertain + fi + # Captain chat is owed the lines past BOTH suppressors: the digest-accounted + # prefix (already on the phone, deliberately not repeated here) and the + # confirmed-delivered prefix (already in chat from an earlier submit whose + # bookkeeping failed). The batch may have grown since that submit, so this is a + # prefix skip, never an all-or-nothing equality check. + chat_delivered=$(away_ledger_chat_delivered "$state" "$batch_id") + chat_start=$accounted + if [ "$chat_delivered" -gt "$chat_start" ]; then + chat_start=$chat_delivered + fi + chat_pending=$(( n - chat_start )) + [ "$chat_pending" -gt 0 ] || chat_pending=0 + chat_body=$(tail -n "+$((chat_start + 1))" "$buf" 2>/dev/null \ + | awk 'NR>1{printf " | "} {printf "%s",$0} END{print ""}') + pointer=$(away_ledger_digest_pointer "$batch_id") + prior_note='' + if [ "$accounted" -gt 0 ] && [ "$tg_status" != accepted ]; then + prior_note=$(printf ' Telegram already delivered and recorded %s earlier event(s) of this batch in %s; do not repeat those in captain chat.' "$accounted" "$pointer") + fi + if [ "$chat_delivered" -gt "$accounted" ] && [ "$tg_status" != accepted ]; then + prior_note="$prior_note$(printf ' The first %s event(s) of this batch were already delivered to captain chat by an earlier submit and are not repeated here.' "$chat_delivered")" + fi + case "$tg_status" in + accepted) + msg=$(printf 'Telegram accepted away-mode alert %s (%s event(s)). Reconcile the durable fleet records now using the private away digests %s; do not repeat this alert in captain chat. (pre-read; re-arm not needed - watcher daemon-managed)' "$tg_id" "$n" "$pointer") + ;; + accepted_undigested) + msg=$(printf 'Telegram accepted away-mode alert %s, but %s event(s) of it are not recorded in a private away digest, so they follow here once: %s. Do not resend these to Telegram; the captain already has them.%s (pre-read; re-arm not needed - watcher daemon-managed)' "$tg_id" "$chat_pending" "$chat_body" "$prior_note") + ;; + uncertain) + msg=$(printf 'Supervisor escalate (%s event(s)): %s (Telegram delivery uncertain; the captain may or may not have received these, so nothing further in this batch goes to the phone.%s Use the existing in-session fallback. Pre-read; re-arm not needed - watcher daemon-managed)' "$chat_pending" "$chat_body" "$prior_note") + ;; + failed|unavailable) + msg=$(printf 'Supervisor escalate (%s event(s)): %s (Telegram delivery %s; the captain was not contacted there.%s Use the existing in-session fallback. Pre-read; re-arm not needed - watcher daemon-managed)' "$chat_pending" "$chat_body" "$tg_status" "$prior_note") + ;; + deferred) + msg=$(printf 'Supervisor escalate (%s event(s)): %s (Telegram delivery deferred; an earlier attempt provably never left this machine and its retry is scheduled, so the captain was not contacted there yet.%s Use the existing in-session fallback. Pre-read; re-arm not needed - watcher daemon-managed)' "$chat_pending" "$chat_body" "$prior_note") + ;; + unknown) + msg=$(printf 'Supervisor escalate (%s event(s)): %s (Telegram delivery unavailable; the away batch ledger is unreadable so nothing was sent to the phone.%s Use the existing in-session fallback. Pre-read; re-arm not needed - watcher daemon-managed)' "$chat_pending" "$chat_body" "$prior_note") + ;; + *) + if [ -n "$prior_note" ]; then + msg=$(printf 'Supervisor escalate (%s event(s)): %s (%s Pre-read; re-arm not needed - watcher daemon-managed)' "$chat_pending" "$chat_body" "${prior_note# }") + else + msg=$(printf 'Supervisor escalate (%s event(s)): %s (pre-read; re-arm not needed - watcher daemon-managed)' "$chat_pending" "$chat_body") + fi + ;; + esac + # The in-session delivery state is recorded in the version before and after the + # submit, as independent fields: opening the attempt states which suffix is in + # flight and never lowers the confirmed prefix, and only a verified submit + # advances that prefix. So neither a failed inject nor a truncation that fails + # AFTER a confirmed submit can become a second delivery of the same lines into + # captain chat: the next flush starts past the confirmed prefix either way. + away_ledger_chat_mark_attempt "$state" "$batch_id" "$chat_start" "$n" \ + || log "WARN: could not record the in-session delivery attempt for this away batch" + if inject_msg "$msg" "$state"; then + if ! away_ledger_chat_mark_confirmed "$state" "$batch_id" "$n"; then + log "ERROR: in-session delivery confirmed but not recorded; a retry may repeat this batch in captain chat" + fi + away_ledger_truncate "$buf" || return 1 + return 0 + fi return 1 } @@ -891,7 +1319,7 @@ inject_wedge_alarm() { # printf 'fm away-mode inject WEDGED: %ss undelivered as of %s\n' "$age" "$(date '+%Y-%m-%dT%H:%M:%S%z')" printf 'The supervisor pane could not accept an escalation. Buffered items:\n' cat "$state/.subsuper-escalations" 2>/dev/null - } 2>/dev/null > "$marker" || true + } 2>/dev/null | away_ledger_wedge_record "$state" || true target="${FM_SUPERVISOR_TARGET:-$FM_SUPERVISOR_TARGET_DEFAULT}" backend="${FM_SUPERVISOR_BACKEND:-$FM_SUPERVISOR_BACKEND_DEFAULT}" # Best-effort status-line flash. tmux's display-message is a client-side OSD @@ -911,12 +1339,12 @@ inject_wedge_alarm() { # fi } -_oldest_line_age() { # -> seconds since the oldest buffered item first arrived (sidecar epoch) - local f=$1 since +_oldest_line_age() { # -> seconds since the oldest buffered item first arrived (ledger epoch) + local f=$1 since epoch [ -s "$f" ] || { echo 999999; return; } since="${f}.since" - if [ -r "$since" ]; then - echo $(( $(_now) - $(cat "$since" 2>/dev/null || echo 0) )) + if [ -r "$since" ] && epoch=$(away_ledger_epoch "$f"); then + echo $(( $(_now) - epoch )) else echo 999999 fi @@ -964,7 +1392,7 @@ housekeeping() { # && [ "$(_file_age "$state/.subsuper-inject-wedged")" -ge "$max_defer" ]; then if escalate_flush "$state"; then log "inject recovered: max-defer flush succeeded after ${oldest}s undelivered" - rm -f "$state/.subsuper-inject-wedged" + away_ledger_wedge_clear "$state" || true else inject_wedge_alarm "$state" "$oldest" fi @@ -994,8 +1422,15 @@ housekeeping() { # case "$?" in 0) rm -f "$marker" ;; 2) rm -f "$marker" ;; - *) escalate_add "$state" "stale persisted ${age}s (possible wedge): $win" - stale_marker_remove "$win" "$state" ;; + *) # The marker is the suppressor this recheck fires from, so it is only + # dropped once the escalation is provably buffered in a committed + # version. A refused append leaves the marker aged, so the next + # housekeeping tick retries this same wedge immediately. + if escalate_add "$state" "stale persisted ${age}s (possible wedge): $win"; then + stale_marker_remove "$win" "$state" + else + log "ERROR: possible-wedge escalation NOT buffered; stale marker left aged so the next tick retries: $win" + fi ;; esac done @@ -1028,8 +1463,11 @@ housekeeping() { # *) last=$(last_status_line "$state/$task.status") if [ -n "$last" ] && status_is_paused "$last"; then - escalate_add "$state" "paused ${age}s (awaiting external, recheck whether the wait still holds): $win" - _now > "$marker" + if escalate_add "$state" "paused ${age}s (awaiting external, recheck whether the wait still holds): $win"; then + _now > "$marker" + else + log "ERROR: pause recheck escalation NOT buffered; pause marker left aged so the next tick retries: $win" + fi else rm -f "$marker" fi @@ -1048,8 +1486,12 @@ housekeeping() { # [ -n "$f" ] || continue seen="$state/.subsuper-seen-status-$(_stale_key "$task")" [ "$(cat "$seen" 2>/dev/null || true)" = "$last" ] && continue - escalate_add "$state" "$(basename "$f"): $last (catch-all scan)" - mark_status_seen "$state" "$task" "$last" + # The seen marker is the suppressor the next scan checks, so it is only + # recorded once the escalation is provably buffered in a committed version. + # A refused append therefore stays re-derivable instead of vanishing. + if escalate_add "$state" "$(basename "$f"): $last (catch-all scan)"; then + mark_status_seen "$state" "$task" "$last" + fi done < <(scan_captain_relevant_statuses "$state") fi } @@ -1202,12 +1644,17 @@ handle_wake() { # case "$action" in escalate) log "escalate: $reason -> $distilled" - escalate_add "$state" "$distilled" - # A terminal-stale escalate must not leave a persistence marker behind, or - # housekeeping re-escalates the same pane as a false wedge later. - [ "$kind" = "stale" ] && stale_marker_remove "$arg" "$state" - mark_escalated_seen "$kind" "$arg" "$state" - [ "${FM_ESCALATE_BATCH_SECS:-$ESCALATE_BATCH_SECS_DEFAULT}" -le 0 ] && { escalate_flush "$state" || true; } + if escalate_add "$state" "$distilled"; then + # A terminal-stale escalate must not leave a persistence marker behind, or + # housekeeping re-escalates the same pane as a false wedge later. Both + # suppressors are only touched once the append is provably committed, so a + # refused append leaves this wake re-derivable by the next scan. + [ "$kind" = "stale" ] && stale_marker_remove "$arg" "$state" + mark_escalated_seen "$kind" "$arg" "$state" + [ "${FM_ESCALATE_BATCH_SECS:-$ESCALATE_BATCH_SECS_DEFAULT}" -le 0 ] && { escalate_flush "$state" || true; } + else + log "ERROR: escalation NOT buffered; stale and seen suppressors left intact so the next scan re-derives it: $distilled" + fi ;; pause) # Declared external-wait pause: record a pause marker (long re-surface @@ -1511,5 +1958,6 @@ else # exported so a real daemon a test later spawns inherits the safe default too. # The executed branch above never runs this, so production is untouched. : "${FM_WEDGE_ALARM_EXEC:=discard}" - export FM_WEDGE_ALARM_EXEC + : "${FM_TG_AWAY_EXEC:=discard}" + export FM_WEDGE_ALARM_EXEC FM_TG_AWAY_EXEC fi diff --git a/bin/fm-tg-followup.sh b/bin/fm-tg-followup.sh index f9293416bf..988ed80d1c 100755 --- a/bin/fm-tg-followup.sh +++ b/bin/fm-tg-followup.sh @@ -154,14 +154,19 @@ fi STRIPPED=$(printf '%s' "$TEXT" | tr -d '[:space:]') [ -n "$STRIPPED" ] || { echo "fm-tg-followup: refusing to send an empty follow-up" >&2; exit 2; } -TG_BIN=${FMTG_TG_BIN:-$HOME/dev/phone-inbox/tg} -if [ ! -f "$TG_BIN" ] || [ ! -x "$TG_BIN" ]; then - echo "fm-tg-followup: phone-inbox tg client not runnable at $TG_BIN" >&2 +# "No runnable client" is decided OUT OF BAND, before the client can run at all, +# so the diagnostic can never be forged by an exit status the client produced for +# its own reasons (a real client that exits 127 is a rejected send, not a missing +# client). Past this guard every non-zero status is a send failure. +if ! fmtg_client_runnable; then + echo "fm-tg-followup: phone-inbox tg client not runnable at $(fmtg_tg_bin)" >&2 exit 1 fi # The text travels on stdin only; tg owns the Telegram message limit. -if ! printf '%s' "$TEXT" | "$TG_BIN"; then +# fmtg_send_stdin keeps every firstmate outbound path on this same client and +# owns resolving it. +if ! printf '%s' "$TEXT" | fmtg_send_stdin; then echo "fm-tg-followup: tg send failed; link kept for retry" >&2 exit 1 fi diff --git a/bin/fm-tg-lib.sh b/bin/fm-tg-lib.sh index 3137beabce..b8fc204797 100644 --- a/bin/fm-tg-lib.sh +++ b/bin/fm-tg-lib.sh @@ -30,6 +30,11 @@ # tg_followups to 0 # fmtg_meta_followups_set - rewrite just the follow-up counter # fmtg_meta_link_clear - remove the Telegram note link entirely +# fmtg_tg_bin - resolved path of that client (FMTG_TG_BIN +# overrides), the single owner of the default +# fmtg_send_stdin - send one message through phone-inbox's +# existing authenticated tg client, with text +# on stdin and no credential handling here FM_TG_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=bin/fm-x-lib.sh @@ -46,6 +51,33 @@ fmtg_inbox_root() { printf '%s\n' "${PHONE_INBOX_ROOT:-$HOME/.claude/inbox}" } +# The phone-inbox tg client remains the single outbound transport owner. +# Callers provide text on stdin only, so alert content never appears in process +# arguments. The client validates Telegram's accepted response and scrubs its +# own diagnostics; callers should still classify failures without relaying raw +# stderr because it can name private local paths. +# Single owner of the client path and its FMTG_TG_BIN override, so no caller has +# to re-derive the default in order to describe a missing client. +fmtg_tg_bin() { + printf '%s\n' "${FMTG_TG_BIN:-$HOME/dev/phone-inbox/tg}" +} + +# Whether the client is runnable at all, checked as its own out-of-band signal +# so a caller can treat "no client to run" as unforgeable and distinct from any +# exit status the client itself might later return once it actually ran. +fmtg_client_runnable() { + local tg_bin + tg_bin=$(fmtg_tg_bin) + [ -f "$tg_bin" ] && [ -x "$tg_bin" ] +} + +fmtg_send_stdin() { + local tg_bin + tg_bin=$(fmtg_tg_bin) + [ -f "$tg_bin" ] && [ -x "$tg_bin" ] || return 127 + "$tg_bin" +} + # Print the numeric ids of pending phone-inbox notes, one per line, sorted and # deduplicated. A pending note is a file named --.md where the # slug is derived from the note BODY, so only the strictly validated numeric id diff --git a/docs/architecture.md b/docs/architecture.md index 514a9c97ed..50db0a6e3d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -75,6 +75,7 @@ Terminal verbs remain captain-relevant, while a nonterminal progress verb cannot The always-on watcher also uses that library's absorb classification on no-verb signals and first-sighting stale panes before status-log terminality is trusted, while the daemon maintains distinct wedge and declared-pause recheck cadences. In away mode, seen-status dedupe does not clear possible-wedge aging for nonterminal progress, so housekeeping still re-escalates an unchanged idle pane at the configured bound. The daemon escalates captain-relevant events, plus a bounded recheck for a declared pause that remains idle, as one batched, single-line digest using the canonical `away-supervisor` kind from `bin/fm-operational-input.sh` so firstmate can distinguish it structurally from real messages. +When Telegram mode is configured, that batch goes to the captain's phone first and an accepted send reduces the injection to a text-free receipt; see [Optional Telegram mode](#optional-telegram-mode). Its supervisor injection path supports tmux and herdr panes, with `FM_SUPERVISOR_BACKEND` and `FM_SUPERVISOR_TARGET` resolved independently from the task-spawn backend. Pane existence, busy checks, composer checks, capture, and verified submit route through `bin/fm-backend.sh`: tmux keeps the same submit core used by the tmux send backend, while herdr uses native busy state, native agent-state submit confirmation on idle baselines, and its ANSI-aware structural composer classifier for pending-input guards and submit fallback. The tmux submit core (shared `fm_tmux_submit_enter_core`) treats a busy pane + retries-exhausted + composer-still-pending as a queued Enter (opencode 1.18.4 accepts Enter mid-turn and queues it for after the turn), reported as `empty` so the daemon and `fm-send` do not re-send; an idle pane keeps the `pending` verdict as a genuine swallow. The same opencode busy-queue case is a known gap on the herdr adapter and is recorded in `docs/herdr-backend.md` rather than patched here. @@ -235,6 +236,7 @@ On the locked session-start bootstrap step the flag creates `state/tg-watch.chec The `fmtg-respond` agent-only skill claims each note through phone-inbox's `inbox claim`, classifies it as work, a question, or a pure acknowledgment, runs actionable reversible requests through firstmate's normal lifecycle, and replies in the same chat through `inbox reply`. Work that spawns a longer-running task is linked by `bin/fm-tg-link.sh` (`tg_note=`, `tg_note_ts=`, `tg_followups=` in that task's meta), and later milestone and completion wakes send up to three bounded follow-ups through `bin/fm-tg-followup.sh`, with `--final` always clearing the link. Every live send notifies the captain's phone, so this mode never sends as a test. +While away mode is active, the sub-supervisor also routes each captain-relevant escalation batch to the phone through that same client, once per batch increment, as notice only that grants no approval; `bin/fm-away-ledger-lib.sh` owns the batch ledger and delivery evidence, and the [Telegram mode configuration reference](configuration.md#telegram-mode-configtelegram-mode) owns the outcome, retry, and fallback contract. ## Project memory belongs to projects diff --git a/docs/configuration.md b/docs/configuration.md index 132c658469..3e320a2e0d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -397,6 +397,24 @@ Completion follow-ups go through `bin/fm-tg-followup.sh`, which sends through th Follow-ups are deliberately not gated on the opt-in flag: removing the flag stops new notes and new links, but a task linked before opt-out may still finish its follow-up thread within the same cap and window, so a request the captain made from his phone always gets its outcome. Every live send notifies the captain's phone, so nothing in this mode sends as a test. +While `state/.afk` is present, the away daemon also routes each captain-relevant escalation batch through the same phone-inbox `tg` client before its in-session delivery. +Each notice carries an explicit reminder that delivery grants no approval for a merge, privileged change, destructive action, or security-sensitive action. +`bin/fm-away-ledger-lib.sh` owns one ledger per batch in `state/.subsuper-escalations.since` - the batch identity plus its `reserved`, `confirmed`, and `accounted` line counts, its attempt ordinal, and its retry schedule - and the away daemon, away start, and away return all query and transition that one owner instead of keeping counters of their own. Older record shapes are migrated in place on read, so upgrading the daemon mid-session keeps the batch's counts rather than failing closed. +A notice covers only the lines past `reserved`, so a batch that grows while its in-session receipt is wedged sends one notice per NEW increment and never repeats a line the captain already has. +An accepted and fully recorded send causes the daemon to inject only a non-secret receipt into Firstmate, pointing at the batch's private away digests instead of repeating the alert contents in captain chat, while preserving Firstmate's obligation to reconcile and act on the durable work records. +Every other outcome carries the not-yet-recorded items into the in-session escalation as the existing fallback and labels the Telegram outcome honestly: `uncertain` (the send may or may not have arrived), `failed` or `unavailable` (nothing left the box, so the captain was definitively not contacted there), an accepted send whose digest is incomplete (the items follow once in session), or an unreadable ledger (nothing was sent to the phone at all). +Only outcomes that prove nothing left the box - a missing client, a watchdog that could not start, or a failure to write the evidence record or outbound spool - retire the attempt: the ledger gives the reservation back, advances the batch's attempt ordinal, and arms a retry schedule in the same atomic write, so the very same lines are offered to the phone again under a fresh delivery id once that schedule elapses, while the retired attempt's evidence record stays durable and auditable. +`FM_TG_AWAY_RETRY_SECS` is that schedule and is honoured only when positive; otherwise one `FM_ESCALATE_BATCH_SECS` window is used, and a non-positive or malformed window falls back to 90 seconds. The delay is therefore never zero, so immediate batching (`FM_ESCALATE_BATCH_SECS=0`) still flushes every tick as documented while a persistent local failure retries on a bounded cadence instead of re-attempting and minting another evidence record on every housekeeping tick; a deferred flush says so in session and contacts nobody. +A failure to write the ledger itself is the one proven-local case that records nothing at all: no reservation was taken and no attempt was made, so publishing an outcome would tombstone a delivery id that never sent anything - the flush reports it in session and the next flush retries the same id cleanly once the state directory is writable. A watchdog timeout, or a client that ran and exited non-zero (the real client cannot distinguish Telegram rejecting the message from a lost response to one it already accepted), keeps the reservation and reports `uncertain`, and because acceptance may only extend a contiguous evidence-proven prefix, that batch sends nothing further to the phone and finishes through the visible in-session fallback. +Delivery evidence lives in the private `state/tg-away-delivery/.status` records as `attempting`, `accepted`, `failed`, or `unavailable` plus an epoch and a non-secret reason class. +Those evidence records never contain alert text, raw sender output, a token, or a chat id. +The separate private `state/tg-away-digest/.items` working records DO hold the distilled escalation lines - the same text the in-session escalation would have carried - so an accepted one-shot escalation such as a stale wedge stays actionable after the buffer is truncated; they carry no token, chat id, or sender output, are 0600, are folded into the return catch-up evidence before they are retired, and retire with the away session. +Every ledger transition - the daemon's escalation append, the phone reservation, the evidence-proven acceptance, a retired attempt and its retry schedule, the digest and the accounted count it advances, the wedge evidence, the in-session truncation, and each away lifecycle transaction (an away entry, a crash rollback) - publishes the whole owned unit (escalation buffer, ledger sidecar, wedge marker, in-session delivery state, and digests) as one complete immutable version under `state/tg-away-versions/v./`, built in a private staging directory under the store's owner lock, validated, and made visible by a single rename, so a crash can never leave a half-built version behind. The in-session delivery state is part of that unit and is recorded before and after the captain-chat submit, so a truncation that fails after a confirmed submit is retried as bookkeeping instead of re-delivering the same batch. The live artifacts are never written as a source of truth: they are the projection of whichever version the store's `active` pointer names, rewritten only after that version is durable, and a superseded predecessor retires as soon as its successor - a complete superset of it - is applied. A version's content reaches the live paths only through the store's single `active` pointer, switched atomically and only while no away daemon holds the supervise-daemon lock, and installing it replaces the live unit wholesale; nothing is ever merged file by file into a live unit or into a published version, so a version's counts can never be paired with a foreign buffer and a running daemon's freshly appended escalation can never be overwritten. A refused switch simply leaves the version retained: the return catch-up folds every retained version's still-owed lines - using each version's own `accounted` offset, so nothing already digested is repeated - and whole versions are retired only once that catch-up is acknowledged. +The ledger reserves lines before network I/O and only durable `accepted` evidence advances `confirmed` - and only over a contiguous prefix, so an accepted later increment can never imply acceptance of an earlier uncertain line - so a restart resolves an interrupted send from the evidence rather than guessing: an `accepted` record confirms it with no second alert, an unresolved `attempting` record is retired as uncertain and never retried, and a reservation with no evidence at all is released so those lines can still reach the phone. +Every send goes through the ledger by construction: `telegram_away_deliver` requires the batch id and the current reserved/total/accounted counts, validates each count on its own before any comparison, sender call, or ledger write, refuses when they do not match the ledger, and mints no delivery id of its own. +A successful `tg` exit means the Telegram API returned an accepted response end to end; it does not prove the captain read the notice. +This away-only use does not change inbound notes, linked-task follow-ups, opt-out behavior, watcher ownership, return catch-up, or buffering outside away mode. + ## Environment variables Runtime tuning via environment variables (defaults shown): @@ -498,7 +516,9 @@ FM_ESCALATE_BATCH_SECS=90 # buffer window for batched escalation digest FM_MAX_DEFER_SECS=300 # max buffered escalation age before retry plus wedge alarm; 0 disables FM_WEDGE_ALARM_CHANNEL= # override config/wedge-alarm with one active-alert directive for the wedge alarm; off|auto|osascript|herdr|command:; absent = auto (macOS -> an OS notification) FM_WEDGE_ALARM_EXEC= # notifier seam: route every channel (osascript, herdr, command:) through this command as ` `; "discard" fires nothing; unset in production; the daemon defaults it to "discard" when sourced so no test posts a real notification (docs/wedge-alarm.md) -FM_WEDGE_ALARM_TIMEOUT_SECS=10 # maximum seconds for each osascript, herdr, override, or command: notifier before its watchdog terminates it and continues to the next channel; invalid or zero values use 10 +FM_WEDGE_ALARM_TIMEOUT_SECS=10 # maximum seconds for each osascript, herdr, override, or command: notifier, and for the away-mode Telegram escalation send, before its watchdog terminates it and continues (a terminated Telegram send falls back to the in-session escalation); invalid or zero values use 10 +FM_TG_AWAY_RETRY_SECS= # seconds a retired proven-local away Telegram attempt waits before the same lines may be offered to the phone again; honoured only when positive, otherwise one FM_ESCALATE_BATCH_SECS window, and a non-positive or invalid window uses 90 - the delay is never 0, so the retry cannot spin per housekeeping tick +FM_TG_AWAY_EXEC= # away-mode Telegram seam: "discard" sends nothing to the phone and reports the batch as off; unset in production; the daemon defaults it to "discard" when sourced so no test reaches the captain's phone FM_INJECT_FAIL_SLEEP=30 # seconds to back off when the supervisor pane is unavailable FM_INJECT_CONFIRM_RETRIES=3 # daemon Enter-retry attempts after typing a digest once FM_INJECT_CONFIRM_SLEEP=0.5 # seconds between daemon submit checks diff --git a/docs/scripts.md b/docs/scripts.md index ec6b11c1e9..f3d3261892 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -93,7 +93,8 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-x-dismiss.sh` | Dismiss a skipped X-mode mention at the relay without replying | | `fm-x-link.sh` | Link a spawned task to its originating X-mode mention in task meta | | `fm-x-followup.sh` | Detect, post, and cap completion follow-ups for an X-mode-linked task | -| `fm-tg-lib.sh` | Shared Telegram-mode opt-in, pending-note, shim-identity, and task-link helpers | +| `fm-away-ledger-lib.sh` | Single owner of the away escalation batch ledger, its private digests, and their retirement | +| `fm-tg-lib.sh` | Shared Telegram-mode opt-in, pending-note, shim-identity, task-link, and outbound `tg` client helpers | | `fm-tg-poll.sh` | One local phone-inbox sweep: offer each pending note id once and emit its wake | | `fm-tg-link.sh` | Link a spawned task to its originating Telegram note in task meta | | `fm-tg-followup.sh` | Detect, send, and cap completion follow-ups for a Telegram-linked task | diff --git a/tests/fm-afk-launch.test.sh b/tests/fm-afk-launch.test.sh index b65bd9cdc2..817c21bdac 100755 --- a/tests/fm-afk-launch.test.sh +++ b/tests/fm-afk-launch.test.sh @@ -230,6 +230,106 @@ unit_failed_start_rolls_back_state() { rm -rf "$st" } +unit_failed_start_preserves_digests_mid_session() { + local st + st=$(mktemp -d "${TMPDIR:-/tmp}/fm-afk-failed-start-digest.XXXXXX") + mkdir -p "$st/state/tg-away-digest" + printf 'was already away\n' > "$st/state/.afk" + printf 'pending\n' > "$st/state/.subsuper-escalations" + printf '1700000000-abc 1 1 1 0 0 x\n' > "$st/state/.subsuper-escalations.since" + printf 'blocked: needs the captain\n' > "$st/state/tg-away-digest/1700000000-abc-a0-0-x.items" + if FM_HOME="$st" FM_STATE_OVERRIDE="$st/state" FM_SUPERVISOR_TARGET=unused \ + FM_SUPERVISOR_BACKEND=unsupported "$LAUNCH" start >/dev/null 2>&1; then + fail "failed start (mid-session): unsupported backend unexpectedly succeeded" + elif [ -f "$st/state/tg-away-digest/1700000000-abc-a0-0-x.items" ] \ + && [ "$(cat "$st/state/tg-away-digest/1700000000-abc-a0-0-x.items")" = 'blocked: needs the captain' ]; then + pass "failed start (mid-session): rollback preserves accepted digests through the owner API" + else + fail "failed start (mid-session): rollback lost an accepted digest" + fi + rm -rf "$st" +} + +unit_failed_start_rolls_back_stale_digests() { + local st + st=$(mktemp -d "${TMPDIR:-/tmp}/fm-afk-failed-start-stale-digest.XXXXXX") + mkdir -p "$st/state/tg-away-digest" + printf 'leftover from a prior session\n' > "$st/state/tg-away-digest/stale-a0-0-x.items" + if FM_HOME="$st" FM_STATE_OVERRIDE="$st/state" FM_SUPERVISOR_TARGET=unused \ + FM_SUPERVISOR_BACKEND=unsupported "$LAUNCH" start >/dev/null 2>&1; then + fail "failed start (fresh entry): unsupported backend unexpectedly succeeded" + elif [ -f "$st/state/tg-away-digest/stale-a0-0-x.items" ]; then + pass "failed start (fresh entry): rollback restores the pre-attempt digest snapshot exactly" + else + fail "failed start (fresh entry): rollback lost the pre-attempt digest snapshot" + fi + rm -rf "$st" +} + +# An away batch version whose active-pointer switch was committed but never +# materialised (a crash mid-rollback) is replayed by the NEXT entry, before any +# daemon can start - and its content survives in the immutable store for the +# return catch-up rather than being discarded by the fresh entry's own clear. +unit_next_entry_replays_a_pending_version_switch() { + local st vdir + st=$(mktemp -d "${TMPDIR:-/tmp}/fm-afk-version-replay.XXXXXX") + vdir="$st/state/tg-away-versions/v.1700000000-pending" + mkdir -p "$st/state" + (umask 077; mkdir -p "$vdir/tg-away-digest") + printf 'blocked: stranded by an interrupted switch\n' > "$vdir/escalations" + printf '1700000000-abc 0 0 0 0 0 none\n' > "$vdir/escalations.since" + printf 'blocked: stranded digest text\n' \ + > "$vdir/tg-away-digest/1700000000-abc-a0-0-x.items" + printf 'escalations\n' > "$vdir/manifest" + : > "$vdir/.complete" + printf 'v.1700000000-pending\n' > "$st/state/tg-away-versions/active" + if FM_HOME="$st" FM_STATE_OVERRIDE="$st/state" FM_AFK_LAUNCH_ENTRY="$SLEEPER" \ + bash -c '. "$1"; fm_afk_launch_start_native' _ "$LAUNCH" >/dev/null 2>&1; then + if grep -rq 'stranded by an interrupted switch' "$st/state/tg-away-versions" 2>/dev/null \ + && grep -rq 'stranded digest text' "$st/state/tg-away-versions" 2>/dev/null \ + && [ ! -e "$st/state/tg-away-versions/active" ]; then + pass "next entry: replays a pending version switch and keeps its content for catch-up" + else + fail "next entry: left the switch pending or lost the stranded version's content" + fi + else + fail "next entry: native start failed unexpectedly" + fi + FM_HOME="$st" FM_STATE_OVERRIDE="$st/state" "$LAUNCH" stop >/dev/null 2>&1 + rm -rf "$st" +} + +# A refresh runs while the daemon is LIVE and is the sole writer of the live +# buffer, so the launcher must perform no pointer switch there: a committed +# pending switch stays pending and the daemon's freshly appended escalation is +# never overwritten by a version's copy. +unit_refresh_never_switches_the_version_pointer() { + local st vdir + st=$(mktemp -d "${TMPDIR:-/tmp}/fm-afk-refresh-no-switch.XXXXXX") + vdir="$st/state/tg-away-versions/v.1700000000-pending" + mkdir -p "$st/state/.supervise-daemon.lock" + (umask 077; mkdir -p "$vdir") + printf '%s\n' "$$" > "$st/state/.supervise-daemon.lock/pid" + printf 'blocked: a retired batch, never adopt me\n' > "$vdir/escalations" + printf 'escalations\n' > "$vdir/manifest" + : > "$vdir/.complete" + printf 'v.1700000000-pending\n' > "$st/state/tg-away-versions/active" + printf 'blocked: appended by the live daemon\n' > "$st/state/.subsuper-escalations" + if FM_HOME="$st" FM_STATE_OVERRIDE="$st/state" bash -c ' + . "$1" + daemon_lock_held_by_live_daemon() { return 0; } + fm_afk_launch_start_native + ' _ "$LAUNCH" >/dev/null 2>&1 \ + && [ "$(cat "$st/state/.subsuper-escalations")" = 'blocked: appended by the live daemon' ] \ + && [ ! -e "$st/state/tg-away-versions/active.applied" ] \ + && [ -e "$vdir/.complete" ]; then + pass "refresh: no pointer switch happens while the daemon is live" + else + fail "refresh: the launcher switched the version pointer under a live daemon" + fi + rm -rf "$st" +} + unit_concurrent_start_serialized() { command -v tmux >/dev/null 2>&1 || { echo "skip: tmux not found (concurrent start)"; return 0; } local st cap_session cap_pane first second rec count @@ -776,20 +876,24 @@ unit_confirmed_absence_succeeds() { rm -rf "$st" } -unit_incomplete_restore_retains_backup() { - local st backup - st=$(mktemp -d "${TMPDIR:-/tmp}/fm-afk-restore-fail.XXXXXX") +# A rollback whose pointer switch is refused or fails must report the failure and +# leave the published version in place: it is the only surviving copy of the +# pre-attempt batch, so the return catch-up folds it instead of losing it. +unit_failed_rollback_retains_the_version() { + local st + st=$(mktemp -d "${TMPDIR:-/tmp}/fm-afk-rollback-retain.XXXXXX") mkdir -p "$st/state" - backup=$(mktemp -d "$st/state/.afk-launch-backup.XXXXXX") - printf 'prior\n' > "$backup/.afk" - if FM_HOME="$st" FM_STATE_OVERRIDE="$st/state" bash -c ' + printf 'blocked: still owed to captain chat\n' > "$st/state/.subsuper-escalations" + FM_HOME="$st" FM_STATE_OVERRIDE="$st/state" bash -c ' . "$1" - cp() { return 1; } - ! fm_afk_launch_restore_backup "$2" 1 - ' _ "$LAUNCH" "$backup" && [ -d "$backup" ] && [ -e "$backup/.afk" ]; then - pass "rollback restore: incomplete restoration retains its recovery backup" + away_ledger_version_activate() { return 3; } + fm_afk_launch_flag_write() { return 1; } + ! fm_afk_launch_start_native + ' _ "$LAUNCH" >/dev/null 2>&1 + if grep -rq 'still owed to captain chat' "$st/state/tg-away-versions" 2>/dev/null; then + pass "rollback: a refused pointer switch retains the version for the catch-up" else - fail "rollback restore: incomplete restoration discarded its backup" + fail "rollback: a refused pointer switch discarded the only copy of the batch" fi rm -rf "$st" } @@ -914,6 +1018,11 @@ unit_fresh_vs_refresh unit_stop_ordering unit_stop_rejects_reused_pid unit_failed_start_rolls_back_state +unit_failed_start_preserves_digests_mid_session +unit_failed_start_rolls_back_stale_digests +unit_next_entry_replays_a_pending_version_switch +unit_refresh_never_switches_the_version_pointer +unit_failed_rollback_retains_the_version unit_concurrent_start_serialized unit_lock_initialization_grace unit_signal_exits_with_lock_cleanup @@ -938,7 +1047,7 @@ unit_stop_confirms_daemon_exit unit_refresh_validates_record unit_clear_failure_aborts_entry unit_confirmed_absence_succeeds -unit_incomplete_restore_retains_backup + unit_flag_write_failure_aborts e2e_herdr e2e_tmux diff --git a/tests/fm-afk-return.test.sh b/tests/fm-afk-return.test.sh index aa3107440b..75dbde4c0e 100755 --- a/tests/fm-afk-return.test.sh +++ b/tests/fm-afk-return.test.sh @@ -19,6 +19,8 @@ install_runner() { # cp "$ROOT/bin/fm-afk-return.sh" "$dir/bin/" cp "$ROOT/bin/fm-wake-lib.sh" "$dir/bin/" cp "$ROOT/bin/fm-classify-lib.sh" "$dir/bin/" + cp "$ROOT/bin/fm-away-ledger-lib.sh" "$dir/bin/" + cp "$ROOT/bin/fm-x-lib.sh" "$dir/bin/" cat > "$dir/bin/fm-afk-launch.sh" <<'SH' #!/usr/bin/env bash [ "${1:-}" = stop ] || exit 2 @@ -209,7 +211,138 @@ test_check_retries_recorded_terminal_teardown() { pass "check retries recorded terminal teardown and keeps catch-up gated until success" } +test_return_folds_accepted_away_digest_before_retiring_it() { + local dir out + dir="$TMP_ROOT/away-digest-fold" + install_runner "$dir" + : > "$dir/home/state/.afk" + mkdir -p "$dir/home/state/tg-away-digest" + chmod 0700 "$dir/home/state/tg-away-digest" + printf 'stale persisted 300s (possible wedge): fm-task-3\n' \ + > "$dir/home/state/tg-away-digest/b1-0-abc.items" + chmod 0600 "$dir/home/state/tg-away-digest/b1-0-abc.items" + + out=$(run_return "$dir" begin) \ + || fail "an accepted away digest must not block the return catch-up: $out" + case "$out" in + *'catch-up away-telegram: stale persisted 300s (possible wedge): fm-task-3'*) ;; + *) fail "return catch-up did not fold the accepted away digest: $out" ;; + esac + [ ! -e "$dir/home/state/tg-away-digest" ] \ + || fail "the away digest must retire once it has been folded into the catch-up" + pass "return catch-up folds every accepted away digest in before retiring it" +} + + +# A line the ledger already accounted for in a private away digest reaches the +# catch-up ONCE. Reproduces the accepted-but-unreceipted case: the phone accepted +# the alert, the in-session receipt injection failed, so the buffer still holds +# the line with accounted == 1. Appending the whole buffer as well as the digest +# would present the same escalation as two separate items to act on. +test_return_evidence_never_duplicates_a_digested_line() { + local dir out rc=0 count + dir="$TMP_ROOT/no-duplicate-digested" + install_runner "$dir" + : > "$dir/home/state/.afk" + printf 'blocked: accepted on the phone, receipt injection failed\n' \ + > "$dir/home/state/.subsuper-escalations" + printf '1700000000-abc 1 1 1 0 0 1700000000-abc-a0-0-d1\n' \ + > "$dir/home/state/.subsuper-escalations.since" + mkdir -p "$dir/home/state/tg-away-digest" + chmod 0700 "$dir/home/state/tg-away-digest" + printf 'blocked: accepted on the phone, receipt injection failed\n' \ + > "$dir/home/state/tg-away-digest/1700000000-abc-a0-0-d1.items" + + out=$(run_return "$dir" begin) || rc=$? + [ "$rc" -eq 0 ] || fail "an accounted-for escalation must not gate the catch-up (rc=$rc): $out" + count=$(printf '%s\n' "$out" | grep -c 'receipt injection failed') + [ "$count" -eq 1 ] \ + || fail "a digested escalation must reach the catch-up exactly once (got $count): $out" + case "$out" in + *'catch-up away-telegram: blocked: accepted on the phone, receipt injection failed'*) ;; + *) fail "the digested escalation must be reported as the away-telegram item: $out" ;; + esac + pass "return catch-up never presents an already-digested escalation twice" +} + +# A failed away-mode stop leaves the daemon possibly alive, so nothing may mutate +# the immutable batch versions: the catch-up only READS them, and retirement waits +# for the acknowledged pass. +test_return_failed_stop_leaves_away_versions_untouched() { + local dir out rc=0 vdir + dir="$TMP_ROOT/failed-stop-versions" + install_runner "$dir" + vdir="$dir/home/state/tg-away-versions/v.1700000000-retained" + mkdir -p "$vdir" + printf 'blocked: retained by a refused pointer switch\n' > "$vdir/escalations" + printf 'escalations\n' > "$vdir/manifest" + : > "$vdir/.complete" + date +%s > "$dir/home/state/.afk" + : > "$dir/home/state/.fail-terminal-stop-once" + + out=$(run_return "$dir" begin) || rc=$? + [ "$rc" -eq 3 ] || fail "a failed away-mode stop must gate the catch-up (rc=$rc): $out" + [ -e "$vdir/.complete" ] \ + || fail "a failed stop must leave every immutable away batch version untouched" + case "$out" in + *'catch-up escalation: blocked: retained by a refused pointer switch'*) ;; + *) fail "a retained away batch version must be folded into the catch-up: $out" ;; + esac + + rc=0 + out=$(run_return "$dir" check) || rc=$? + [ "$rc" -eq 0 ] || fail "check must close the gate once the stop succeeds (rc=$rc): $out" + [ ! -e "$dir/home/state/tg-away-versions" ] \ + || fail "away batch versions must retire only once the catch-up is acknowledged" + pass "a failed stop never mutates away batch versions; acknowledgement retires them" +} + + +# Retirement is part of acknowledgement, so a version that could not be removed +# keeps the gate open instead of reporting a clear catch-up over content the next +# pass would fold again. The retry must not duplicate that evidence either. +test_return_keeps_the_gate_until_versions_retire() { + local dir out rc=0 vdir count + dir="$TMP_ROOT/retire-fault" + install_runner "$dir" + vdir="$dir/home/state/tg-away-versions/v.1700000000-stuck" + mkdir -p "$vdir" + printf 'blocked: retained and not yet acknowledged\n' > "$vdir/escalations" + printf 'escalations\n' > "$vdir/manifest" + : > "$vdir/.complete" + chmod 500 "$vdir" + + out=$(run_return "$dir" begin) || rc=$? + [ "$rc" -eq 3 ] || fail "a failed away batch retirement must gate the catch-up (rc=$rc): $out" + [ -e "$dir/home/state/.afk-return-catchup" ] \ + || fail "a failed retirement left no gate behind" + case "$out" in + *'away batch records could not be retired'*) ;; + *) fail "a failed retirement must surface a lifecycle blocker: $out" ;; + esac + [ -e "$vdir/.complete" ] || fail "the version must survive a failed retirement intact" + + rc=0 + out=$(run_return "$dir" check) || rc=$? + [ "$rc" -eq 3 ] || fail "check must keep gating while retirement still fails (rc=$rc): $out" + count=$(grep -c 'retained and not yet acknowledged' "$dir/home/state/.afk-return-catchup") + [ "$count" -eq 1 ] \ + || fail "re-folding the same surviving version must not duplicate its evidence (got $count)" + + chmod 700 "$vdir" + rc=0 + out=$(run_return "$dir" check) || rc=$? + [ "$rc" -eq 0 ] || fail "check must close the gate once retirement succeeds (rc=$rc): $out" + [ ! -e "$dir/home/state/tg-away-versions" ] \ + || fail "an acknowledged catch-up must leave no away batch version behind" + pass "the return gate stays open until whole-version retirement succeeds, without duplicating evidence" +} + test_return_gate_orders_catchup_before_bearings +test_return_folds_accepted_away_digest_before_retiring_it +test_return_evidence_never_duplicates_a_digested_line +test_return_failed_stop_leaves_away_versions_untouched +test_return_keeps_the_gate_until_versions_retire test_explicit_reclassification_requires_durable_reason test_captain_decision_does_not_masquerade_as_firstmate_blocker test_away_reentry_refuses_pending_return_gate diff --git a/tests/fm-bearings-skill.test.sh b/tests/fm-bearings-skill.test.sh new file mode 100755 index 0000000000..4fd9d485d3 --- /dev/null +++ b/tests/fm-bearings-skill.test.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Static regression tests for the internal /bearings skill contract. +# shellcheck disable=SC2016 +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +BEARINGS_SKILL="$ROOT/.agents/skills/bearings/SKILL.md" +README="$ROOT/README.md" +AUDIENCES="$ROOT/docs/documentation-audiences.json" + +skill_body() { + awk 'BEGIN { seen = 0 } /^---$/ { seen += 1; next } seen >= 2 { print }' "$BEARINGS_SKILL" +} + +chat_contract() { + awk '/^## Chat-response contract$/{capture=1; next} capture && /^## /{exit} capture' "$BEARINGS_SKILL" +} + +test_status_skill_is_absent() { + assert_absent "$ROOT/.agents/skills/status" "internal status skill directory must be removed" + assert_absent "$ROOT/.agents/skills/status/SKILL.md" "internal status skill file must be removed" + assert_absent "$ROOT/skills/status" "public status skill directory must not exist" + assert_no_grep '.agents/skills/status/SKILL.md' "$AUDIENCES" "documentation audience inventory still lists status" + assert_no_grep '| `/status`' "$README" "README still lists /status" + pass "/status is absent from the skill and documentation surfaces" +} + +test_plain_bearings_is_chat_only_by_default() { + assert_grep 'Plain `/bearings` returns only the concise four-section chat digest.' "$BEARINGS_SKILL" \ + "plain bearings default is not chat-only" + assert_grep 'Plain `/bearings` gathers a fresh bounded snapshot and renders the four-section chat digest without creating, deleting, reading, or replacing `data/status-report-.md`.' "$BEARINGS_SKILL" \ + "plain bearings does not forbid dated report writes" + assert_grep 'Plain mode stops here and writes no report artifact.' "$BEARINGS_SKILL" \ + "plain bearings can continue into report writing" + pass "plain /bearings is chat-only and forbids report artifacts" +} + +test_file_mode_owns_prior_report_artifact_behavior() { + assert_grep 'Only `/bearings file` writes the dated markdown report artifact and then returns the concise four-section chat digest linked to that report.' "$BEARINGS_SKILL" \ + "file mode is not the only report-writing mode" + assert_grep 'Write the full report to `data/status-report-.md` using today'"'"'s date.' "$BEARINGS_SKILL" \ + "file mode does not write the dated report" + assert_grep 'If today'"'"'s file already exists, delete it first, then create a new file from scratch.' "$BEARINGS_SKILL" \ + "file mode does not replace today's report from scratch" + assert_grep 'This is the only write allowed by the skill.' "$BEARINGS_SKILL" \ + "file mode write boundary is missing" + assert_grep 'After writing the file, return the concise four-section chat digest and include the report path or link without adding a fifth section.' "$BEARINGS_SKILL" \ + "file mode does not return the linked four-section digest" + pass "/bearings file owns the prior dated-report behavior" +} + +test_file_option_is_explicit_and_prs_compose() { + assert_grep 'Treat `file` only as an explicit invocation option in the slash command.' "$BEARINGS_SKILL" \ + "file is not pinned to an explicit slash option" + assert_grep 'Do not treat natural-language requests such as "write a report", "save this", "persist it", or "make a file" as file mode unless the invocation explicitly includes the standalone `file` option.' "$BEARINGS_SKILL" \ + "file mode can be triggered fuzzily" + assert_grep 'When the captain asks to include PRs, pass the snapshot command'"'"'s live-PR opt-in.' "$BEARINGS_SKILL" \ + "live PR opt-in is missing" + assert_grep '`/bearings include PRs` remains chat-only and makes the live-PR opt-in.' "$BEARINGS_SKILL" \ + "include PRs does not compose with chat-only mode" + assert_grep '`/bearings file include PRs` writes the dated report and makes the live-PR opt-in.' "$BEARINGS_SKILL" \ + "include PRs does not compose with file mode" + pass "file is explicit and live PR enrichment composes with both modes" +} + +test_single_fresh_snapshot_source_and_authoritative_provenance() { + local body count + body=$(skill_body) + count=$(grep -cF 'bin/fm-bearings-snapshot.sh' "$BEARINGS_SKILL") + [ "$count" = 1 ] || fail "bearings should reference the snapshot owner exactly once, found $count" + assert_contains "$body" 'Run `bin/fm-bearings-snapshot.sh` at invocation time and read its compact output.' \ + "bearings does not gather a fresh snapshot at invocation time" + assert_contains "$body" 'It is the single bounded, deterministic fleet-state source for Bearings and renders TOON by default.' \ + "bearings does not name the single bounded source" + assert_contains "$body" 'Do not create or consult a second fleet-state reader, parser contract, status-event-tail interpretation, visible-session recap, ad-hoc project probe, or ad-hoc `gh-axi`/`gh` query.' \ + "bearings allows a second reader or ad-hoc probe" + assert_contains "$body" 'For registered secondmates, use the snapshot'"'"'s structured-home classification and provenance.' \ + "bearings does not use structured secondmate provenance" + assert_contains "$body" 'Structured captain-held decisions come from `decision-hold-lifecycle` and appear under `decisions_open`.' \ + "bearings does not preserve structured captain-held decisions" + assert_not_contains "$body" 'fm-fleet-snapshot.sh' "bearings creates a second canonical snapshot path" + assert_not_contains "$body" 'fm-crew-state.sh' "bearings creates an extra current-state reader" + pass "bearings keeps the fresh structured snapshot as the single source" +} + +test_chat_contract_four_sections_for_both_modes() { + local body headings expected report_headings + body=$(chat_contract) + headings=$(printf '%s\n' "$body" | sed -nE "s/^[0-9]+\. \*\*([^*]+)\*\*.*/\1/p") + expected=$(printf '%s\n' "Captain's Call" "Recently Landed" "Underway" "Charted Next") + [ "$headings" = "$expected" ] || fail "chat contract must contain exactly four numbered sections in fixed order, got: $headings" + assert_contains "$body" "Nothing needs your action right now" "Captain's Call empty-state sentence" + assert_contains "$body" "No recent completions are in the current baseline" "Recently Landed empty-state sentence" + assert_contains "$body" "Nothing is underway" "Underway empty-state sentence" + assert_contains "$body" "Nothing is queued" "Charted Next empty-state sentence" + report_headings=$(sed -nE 's/^ - \*\*(Captain.s Call|Recently Landed|Underway|Charted Next)\*\*.*/\1/p' "$BEARINGS_SKILL") + [ "$report_headings" = "$expected" ] || fail "detailed report contract must contain the same four complete sections, got: $report_headings" + assert_contains "$body" "no At Anchor section" "the At Anchor exclusion must be documented" + assert_contains "$body" "Every chat digest and file-mode report is a complete current snapshot" "both modes must be complete current snapshots" + assert_contains "$body" "Detailed decisions, plans, full gate reasons, and evidence belong in the file only when file mode is explicit" \ + "plain chat must not depend on a detailed report file" + assert_contains "$body" "In file mode, include the report path or link inside the four-section digest without adding another heading." \ + "file mode must link the report without a fifth section" + pass "both Bearings modes keep the exact four-section chat contract" +} + +test_readme_describes_bearings_modes() { + assert_grep '| `/bearings` | Generate a concise four-section chat digest from bounded local fleet and registered-secondmate state; use `/bearings file` to also replace today'"'"'s dated report in `data/`, and add `include PRs` when live PR enrichment is wanted |' "$README" \ + "README skill table does not describe chat-only default and file option" + assert_grep '- `/bearings` returns the fresh four-section digest in chat only.' "$README" \ + "README lacks plain bearings example" + assert_grep '- `/bearings include PRs` keeps chat-only mode and opts into live PR enrichment.' "$README" \ + "README lacks chat-only live PR example" + assert_grep '- `/bearings file` replaces today'"'"'s `data/status-report-.md` from scratch and links it from the four-section chat digest.' "$README" \ + "README lacks file mode example" + assert_grep '- `/bearings file include PRs` combines the dated report with live PR enrichment.' "$README" \ + "README lacks file mode live PR example" + pass "README documents Bearings default and file mode" +} + +test_status_skill_is_absent +test_plain_bearings_is_chat_only_by_default +test_file_mode_owns_prior_report_artifact_behavior +test_file_option_is_explicit_and_prs_compose +test_single_fresh_snapshot_source_and_authoritative_provenance +test_chat_contract_four_sections_for_both_modes +test_readme_describes_bearings_modes diff --git a/tests/fm-bearings-snapshot.test.sh b/tests/fm-bearings-snapshot.test.sh index f8afefa255..32c1e1b1e9 100755 --- a/tests/fm-bearings-snapshot.test.sh +++ b/tests/fm-bearings-snapshot.test.sh @@ -1863,8 +1863,8 @@ EOF # The /bearings skill is the one owner of the four-section chat-response contract. # Assert it states exactly the four fixed sections in order, each with its explicit -# empty-state sentence, documents the At Anchor exclusion, and mandates a chat that is -# materially shorter than and links to the report file. +# empty-state sentence, documents the At Anchor exclusion, and keeps file-mode links +# inside the four-section digest. test_chat_contract_four_sections() { local skill body headings report_headings expected skill="$ROOT/.agents/skills/bearings/SKILL.md" @@ -1885,8 +1885,8 @@ test_chat_contract_four_sections() { assert_contains "$(cat "$skill")" 'Never read an earlier `data/status-report-*.md`' "prior reports must not influence current output" assert_contains "$(cat "$skill")" "bounded current recent-completions baseline" "Recently Landed must be a current baseline" assert_contains "$body" "no At Anchor section" "the At Anchor exclusion must be documented" - assert_contains "$body" "materially shorter" "the chat must be materially shorter than the report file" - assert_contains "$body" "links to" "the chat must link to the report file" + assert_contains "$body" "materially shorter" "the file-mode chat must be materially shorter than the report file" + assert_contains "$body" "report path or link" "file mode must link the report from inside the digest" pass "the /bearings skill states the four-section chat contract in order, with empty-states and the At Anchor exclusion" } diff --git a/tests/fm-daemon.test.sh b/tests/fm-daemon.test.sh index 90b32a6ef9..7dfccd3844 100755 --- a/tests/fm-daemon.test.sh +++ b/tests/fm-daemon.test.sh @@ -23,6 +23,57 @@ fi TMP_ROOT=$(fm_test_tmproot fm-daemon-tests) +test_afk_start_refuses_to_clear_an_uncaptured_away_batch() { + local dir state out status + dir=$(make_supercase afk-start-uncapturable-batch) + state="$dir/state" + printf 'blocked: crashed away session, never flushed\n' > "$state/.subsuper-escalations" + printf '1700000000-abc 0 0 0 0 0 none\n' > "$state/.subsuper-escalations.since" + # The version store path is occupied by a regular file, so the entry-boundary + # capture cannot publish the predecessor unit as an immutable version. + printf 'not a directory\n' > "$state/tg-away-versions" + + out=$(FM_STATE_OVERRIDE="$state" FM_SUPERVISOR_BACKEND=unsupported "$AFK_START" 2>&1) + status=$? + + [ "$status" -ne 0 ] \ + || fail "a direct away start must refuse when the prior batch cannot be captured" + assert_not_contains "$out" "starting supervise daemon" \ + "a direct away start continued into daemon startup without capturing the prior batch" + assert_grep 'blocked: crashed away session, never flushed' "$state/.subsuper-escalations" \ + "a direct away start cleared un-flushed escalation lines with no surviving version" + pass "a direct away start refuses to clear a prior away batch it could not capture" +} + +# The clear is an owner transaction, so it can refuse. Under this script's errexit +# an unguarded call would end the start silently, mid-retirement, with away mode +# already flagged and no daemon supervising it. +test_afk_start_refuses_when_the_prior_batch_cannot_be_retired() { + local dir state out status + dir=$(make_supercase afk-start-unretirable-batch) + state="$dir/state" + printf 'blocked: crashed away session, never flushed\n' > "$state/.subsuper-escalations" + + out=$(FM_STATE_OVERRIDE="$state" FM_SUPERVISOR_BACKEND=unsupported bash -c ' + . "$1" + away_ledger_retire_batch() { return 1; } + rc=0 + fm_afk_start_main || rc=$? + exit "$rc" + ' _ "$AFK_START" 2>&1) + status=$? + + [ "$status" -ne 0 ] \ + || fail "a direct away start must refuse when the prior batch cannot be retired" + assert_not_contains "$out" "starting supervise daemon" \ + "a direct away start continued into daemon startup after a refused retirement" + assert_contains "$out" "refusing to start the daemon" \ + "a refused retirement must say so instead of ending the run silently" + assert_absent "$state/.afk" \ + "a refused retirement must roll back the away flag this invocation wrote" + pass "a direct away start refuses honestly when the prior batch cannot be retired" +} + test_afk_start_refuses_when_flag_cannot_be_written() { local dir state out status dir=$(make_supercase afk-start-flag-unwritable) @@ -296,6 +347,76 @@ test_housekeeping_paused_resurfaces_and_resets() { pass "housekeeping re-surfaces a stale declared pause on the long cadence and resets its window" } +# Every escalation producer may drop or reset its suppressor only once the +# versioned append has provably committed. The stale marker is the suppressor the +# possible-wedge recheck fires from, so a refused append must leave it aged and +# the next tick must retry the same wedge. +test_housekeeping_refused_append_keeps_the_stale_marker() { + local dir state fakebin win pane key stamp + dir=$(make_supercase stale-refused-append) + state="$dir/state"; fakebin="$dir/fakebin" + win="sess:fm-wedged-w21"; pane="$dir/pane.txt" + printf 'working: still chewing on the migration\n' > "$state/wedged-w21.status" + printf 'idle prompt $\n' > "$pane" + key=$(printf '%s' "wedged-w21" | tr ':/.' '___') + stamp=$(( $(date +%s) - 5000 )) + echo "$stamp" > "$state/.subsuper-stale-$key" + + ( + escalate_add() { return 1; } + log() { :; } + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$win" FM_FAKE_TMUX_CAPTURE="$pane" \ + FM_STATE_OVERRIDE="$state" FM_STALE_ESCALATE_SECS=240 housekeeping "$state" + ) >/dev/null 2>&1 + [ -e "$state/.subsuper-stale-$key" ] \ + || fail "a refused append dropped the stale marker the wedge recheck fires from" + [ "$(cat "$state/.subsuper-stale-$key")" = "$stamp" ] \ + || fail "a refused append must leave the stale marker aged, not re-armed" + [ ! -s "$state/.subsuper-escalations" ] \ + || fail "a refused append must leave nothing buffered" + + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$win" FM_FAKE_TMUX_CAPTURE="$pane" \ + FM_STATE_OVERRIDE="$state" FM_STALE_ESCALATE_SECS=240 housekeeping "$state" + grep -q 'possible wedge' "$state/.subsuper-escalations" \ + || fail "the next tick did not retry the wedge the refused append dropped" + [ -e "$state/.subsuper-stale-$key" ] \ + && fail "a committed append must drop the stale marker" + pass "a refused append leaves the possible-wedge recheck retryable on the next tick" +} + +# Same rule on the pause re-surface path: a refused append must not re-arm the +# pause window, or the forgotten pause rots for another full cadence. +test_housekeeping_refused_append_keeps_the_pause_marker_aged() { + local dir state fakebin win pane key stamp + dir=$(make_supercase paused-refused-append) + state="$dir/state"; fakebin="$dir/fakebin" + win="sess:fm-held-w21"; pane="$dir/pane.txt" + printf 'paused: holding for the upstream tool release\n' > "$state/held-w21.status" + printf 'idle prompt $\n' > "$pane" + key=$(printf '%s' "held-w21" | tr ':/.' '___') + stamp=$(( $(date +%s) - 5000 )) + echo "$stamp" > "$state/.subsuper-paused-$key" + + ( + escalate_add() { return 1; } + log() { :; } + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$win" FM_FAKE_TMUX_CAPTURE="$pane" \ + FM_STATE_OVERRIDE="$state" FM_PAUSE_RESURFACE_SECS=240 housekeeping "$state" + ) >/dev/null 2>&1 + [ "$(cat "$state/.subsuper-paused-$key" 2>/dev/null)" = "$stamp" ] \ + || fail "a refused append re-armed the pause window instead of leaving it aged" + [ ! -s "$state/.subsuper-escalations" ] \ + || fail "a refused append must leave nothing buffered" + + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$win" FM_FAKE_TMUX_CAPTURE="$pane" \ + FM_STATE_OVERRIDE="$state" FM_PAUSE_RESURFACE_SECS=240 housekeeping "$state" + grep -F 'awaiting external' "$state/.subsuper-escalations" >/dev/null 2>&1 \ + || fail "the next tick did not retry the pause recheck the refused append dropped" + [ "$(cat "$state/.subsuper-paused-$key" 2>/dev/null)" != "$stamp" ] \ + || fail "a committed append must reset the pause window" + pass "a refused append leaves the pause re-surface retryable on the next tick" +} + # A pause whose pane became busy again (the crew resumed) drops its marker without # escalating, exactly like a resumed wedge. test_housekeeping_paused_resumed_cleared() { @@ -1766,6 +1887,8 @@ test_inject_msg_defers_on_unrecognized_composer_state() { } test_afk_start_refuses_when_flag_cannot_be_written +test_afk_start_refuses_to_clear_an_uncaptured_away_batch +test_afk_start_refuses_when_the_prior_batch_cannot_be_retired test_afk_start_ignores_stale_pidfile_without_lock test_afk_start_reclaims_stale_daemon_lock_reused_pid test_daemon_state_root_uses_fm_home @@ -1784,6 +1907,8 @@ test_housekeeping_seeds_pause_marker_from_status test_housekeeping_persistent_stale_escalates test_housekeeping_resumed_stale_cleared test_housekeeping_paused_resurfaces_and_resets +test_housekeeping_refused_append_keeps_the_stale_marker +test_housekeeping_refused_append_keeps_the_pause_marker_aged test_housekeeping_paused_resumed_cleared test_housekeeping_paused_unpaused_cleared test_housekeeping_stale_marker_transitions_to_pause diff --git a/tests/fm-tg-mode.test.sh b/tests/fm-tg-mode.test.sh old mode 100755 new mode 100644 index f02b6a3802..d0a6cbc789 --- a/tests/fm-tg-mode.test.sh +++ b/tests/fm-tg-mode.test.sh @@ -21,6 +21,8 @@ set -u # shellcheck source=/dev/null . "$ROOT/bin/fm-tg-lib.sh" +# shellcheck source=/dev/null +. "$ROOT/bin/fm-supervise-daemon.sh" TMP_ROOT=$(fm_test_tmproot fm-tg-mode-tests) @@ -497,6 +499,1725 @@ test_supervision_needed_by_tg_shim() { pass "fm_supervision_status counts the Telegram poll as supervision-needed" } +# ledger_field : assert one ledger count. +ledger_field() { + local state=$1 idx=$2 want=$3 msg=$4 got + got=$(away_ledger_read "$state/.subsuper-escalations" | cut -d' ' -f"$idx") + [ "$got" = "$want" ] || fail "$msg (ledger field $idx = $got, want $want)" +} + +# away_deliver [tg-bin] [fake-tg-exit]: drive telegram_away_deliver +# exactly the way escalate_flush does - read the ledger, hand it the pending body +# plus the current reserved/total/accounted counts and the batch id. Nothing may +# reach the phone without those, so tests go through the same door. +away_deliver() { + local home=$1 state=$2 tg_bin=${3:-} fake_exit=${4:-0} + local buf rec id reserved accounted did n pending + buf="$state/.subsuper-escalations" + rec=$(away_ledger_read "$buf") + IFS=' ' read -r id reserved _ accounted did <<< "$rec" + n=$(( $(wc -l < "$buf" 2>/dev/null || echo 0) )) + pending=$(tail -n "+$((reserved + 1))" "$buf" 2>/dev/null \ + | awk 'NR>1{printf " | "} {printf "%s",$0} END{print ""}') + ( + # shellcheck disable=SC2030,SC2031 # The exports are deliberately scoped to this subshell. + export FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live + # shellcheck disable=SC2030,SC2031 # The exports are deliberately scoped to this subshell. + export FMTG_TG_BIN="$tg_bin" FAKE_TG_EXIT="$fake_exit" + telegram_away_deliver "$state" "$pending" "$reserved" "$n" "$accounted" "$id" + ) +} + +test_away_delivery_is_inert_outside_away_mode() { + local home state tg out + home=$(make_home away-inactive) + state="$home/state" + tg=$(make_fake_tg "$home") + printf '1700000000\n' > "$state/.subsuper-escalations.since" + + out=$(FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" telegram_away_deliver "$state" 'blocked: should stay in session') + [ "$out" = 'off|none' ] || fail "Telegram away delivery must be off outside away mode (got: $out)" + assert_absent "$home/tg-sent.log" "Telegram delivery outside away mode must not call the outbound client" + assert_absent "$state/tg-away-delivery" "Telegram delivery outside away mode must not create evidence" + pass "Telegram escalation delivery is inert outside away mode" +} + +test_away_delivery_is_accepted_once_and_records_no_content() { + local home state tg out first_id second_id + home=$(make_home away-accepted) + state="$home/state" + tg=$(make_fake_tg "$home") + : > "$state/.afk" + escalate_add "$state" 'needs-decision: approve privileged cutover' + + out=$(away_deliver "$home" "$state" "$tg") + first_id=${out#*|} + [ "${out%%|*}" = accepted ] || fail "away delivery must report accepted (got: $out)" + + # A repeat of the very same (batch, offset, body) delivery must re-derive the + # same id, settle from the existing accepted evidence, and never produce a + # second phone alert. + away_ledger_write "$state/.subsuper-escalations" \ + "$(printf '%s' "$first_id" | cut -d- -f1-2)" 0 0 0 0 0 none \ + || fail "could not rewind the ledger for the repeat-delivery case" + out=$(away_deliver "$home" "$state" "$tg") + second_id=${out#*|} + [ "${out%%|*}" = accepted ] || fail "accepted away delivery must remain accepted (got: $out)" + [ "$first_id" = "$second_id" ] || fail "the same away batch must keep one delivery id" + [ "$(grep -c '^---$' "$home/tg-sent.log")" -eq 1 ] \ + || fail "the same away batch reached the outbound client more than once" + grep -Eq '^accepted [0-9]+$' "$state/tg-away-delivery/$first_id.status" \ + || fail "accepted delivery must have non-secret durable evidence" + assert_no_grep 'privileged cutover' "$state/tg-away-delivery/$first_id.status" \ + "delivery evidence must not contain message text" + pass "away delivery records Telegram acceptance and sends each batch exactly once" +} + +test_away_delivery_failure_classes_and_safe_chat_fallback() { + local home state tg out injected + home=$(make_home away-failure) + state="$home/state" + tg=$(make_fake_tg "$home") + : > "$state/.afk" + + # A client that RAN and exited non-zero cannot be distinguished from a lost + # response to an accepted send, so it is uncertain, not failed, and its + # reservation is kept. + escalate_add "$state" 'blocked: credential needed' + out=$(away_deliver "$home" "$state" "$tg" 1) + [ "${out%%|*}" = uncertain ] \ + || fail "a client that ran and rejected must report uncertain (got: $out)" + grep -Eq '^failed [0-9]+ sender_uncertain_response$' "$state/tg-away-delivery/${out#*|}.status" \ + || fail "an uncertain delivery must have non-secret durable evidence" + ledger_field "$state" 2 1 "an uncertain send must keep its phone reservation" + ledger_field "$state" 3 0 "an uncertain send must not be confirmed" + ledger_field "$state" 5 0 "an ambiguous outcome must never retire its attempt for retry" + + # A missing client provably never left the box: unavailable, reservation released. + : > "$state/.subsuper-escalations" + away_ledger_retire "$state/.subsuper-escalations" + escalate_add "$state" 'blocked: second credential needed' + out=$(away_deliver "$home" "$state" "$home/missing-tg") + [ "${out%%|*}" = unavailable ] || fail "a missing Telegram client must report unavailable (got: $out)" + grep -Eq '^unavailable [0-9]+ sender_missing$' "$state/tg-away-delivery/${out#*|}.status" \ + || fail "unavailable delivery must have non-secret durable evidence" + ledger_field "$state" 2 0 "a send that never left the box must release its reservation" + ledger_field "$state" 5 1 "a proven-local failure must retire its attempt for retry" + + : > "$state/.subsuper-escalations" + away_ledger_retire "$state/.subsuper-escalations" + escalate_add "$state" 'blocked: safe fallback required' + injected="$home/injected.log" + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" > "$injected"; return 0; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$home/missing-tg" escalate_flush "$state" + ) || fail "failed Telegram delivery must retain the in-session fallback" + assert_grep 'blocked: safe fallback required' "$injected" \ + "the fallback must preserve the captain-relevant escalation" + assert_grep 'Telegram delivery unavailable; the captain was not contacted there' "$injected" \ + "the fallback must never claim Telegram delivery" + pass "away delivery distinguishes failed and unavailable transport and falls back safely" +} + +test_away_delivery_acceptance_does_not_duplicate_alert_in_chat() { + local home state tg injected + home=$(make_home away-no-chat-duplicate) + state="$home/state" + tg=$(make_fake_tg "$home") + : > "$state/.afk" + printf 'needs-decision: approve privileged cutover\n' > "$state/.subsuper-escalations" + printf '1700000004\n' > "$state/.subsuper-escalations.since" + injected="$home/injected.log" + + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" > "$injected"; return 0; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" escalate_flush "$state" + ) || fail "accepted Telegram delivery must complete the away flush" + + assert_grep 'Telegram accepted away-mode alert' "$injected" \ + "Firstmate must receive a non-secret accepted receipt" + assert_no_grep 'approve privileged cutover' "$injected" \ + "an accepted Telegram alert must not be duplicated into captain chat" + assert_grep 'needs-decision: approve privileged cutover' "$home/tg-sent.log" \ + "the existing outbound client must receive the actual batched alert" + assert_grep 'does not approve a merge, privileged change, destructive action, or security-sensitive action' "$home/tg-sent.log" \ + "the phone notice must preserve approval boundaries" + [ ! -s "$state/.subsuper-escalations" ] || fail "confirmed receipt injection must clear the away buffer" + pass "accepted away alerts reach Telegram once without duplicating their contents in captain chat" +} + +test_away_batch_never_resends_accepted_events() { + local home state tg injected sends pointer digests digest delivery_id + home=$(make_home away-grown-batch) + state="$home/state" + tg=$(make_fake_tg "$home") + : > "$state/.afk" + injected="$home/injected.log" + + # First event: Telegram accepts it, but the in-session receipt is wedged, so + # the buffer and its batch identity are preserved for retry. + printf 'stale persisted 300s (possible wedge): fm-task-3\n' > "$state/.subsuper-escalations" + printf '1700000010\n' > "$state/.subsuper-escalations.since" + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { return 1; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" escalate_flush "$state" + ) && fail "a wedged receipt must leave escalate_flush unconfirmed" + assert_grep 'fm-task-3' "$home/tg-sent.log" "the first event must reach the phone" + + # A second event appends while the receipt is still wedged. The phone must + # receive ONLY the new event, never a repeat of the accepted one. + printf 'needs-decision: approve privileged cutover\n' >> "$state/.subsuper-escalations" + cp "$state/.subsuper-escalations" "$home/expected-items.txt" + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" > "$injected"; return 0; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" escalate_flush "$state" + ) || fail "the grown away batch must flush once the receipt lands" + sends=$(grep -c '^---$' "$home/tg-sent.log") + [ "$sends" -eq 2 ] || fail "the grown batch must reach the phone once per new event (got $sends sends)" + [ "$(grep -c 'fm-task-3' "$home/tg-sent.log")" -eq 1 ] \ + || fail "an already-accepted away event must never be sent to the phone twice" + assert_grep 'approve privileged cutover' "$home/tg-sent.log" \ + "the newly appended event must reach the phone" + assert_no_grep 'approve privileged cutover' "$injected" \ + "an accepted away alert must not be duplicated into captain chat" + [ ! -s "$state/.subsuper-escalations" ] || fail "the confirmed receipt must clear the away buffer" + + # The receipt must point at digests that actually exist and, together, hold + # EVERY accepted item of the grown batch - not just the last delivery's - so a + # one-shot escalation stays actionable after the buffer is truncated. + pointer=$(sed -n 's|.*private away digests state/tg-away-digest/\([^ ;]*\)\.items.*|\1|p' "$injected") + [ -n "$pointer" ] || fail "the accepted receipt must point at the batch digests" + digests=$(eval ls "$state/tg-away-digest/$pointer.items" 2>/dev/null || true) + [ -n "$digests" ] || fail "the receipt's digest pointer must match real files ($pointer)" + for digest in $digests; do + [ -f "$digest" ] && [ ! -L "$digest" ] || fail "a pointed-at digest must be a regular file" + case "$(ls -l "$digest")" in + -rw-------*) ;; + *) fail "the away digest must be private (0600)" ;; + esac + done + # shellcheck disable=SC2086 + [ "$(cat $digests | sort)" = "$(sort "$home/expected-items.txt")" ] \ + || fail "the pointed-at digests must hold exactly the accepted items of the batch" + delivery_id=$(sed -n 's/^Telegram accepted away-mode alert \([^ ,]*\).*/\1/p' "$injected") + assert_no_grep 'privileged cutover' "$state/tg-away-delivery/$delivery_id.status" \ + "delivery evidence must stay text-free" + + # A later failure must not repeat the events Telegram already delivered. + printf 'blocked: credential needed\n' > "$state/.subsuper-escalations" + printf '1700000020\n' > "$state/.subsuper-escalations.since" + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { return 1; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" escalate_flush "$state" + ) && fail "a wedged receipt must leave the second batch unconfirmed" + printf 'paused 600s (awaiting external): fm-task-9\n' >> "$state/.subsuper-escalations" + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" > "$injected"; return 0; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" FAKE_TG_EXIT=1 escalate_flush "$state" + ) || fail "a failed Telegram send must still complete the in-session fallback" + assert_grep 'fm-task-9' "$injected" \ + "the undelivered event must reach the in-session fallback" + assert_no_grep 'credential needed' "$injected" \ + "the fallback must not repeat an event Telegram already delivered" + pass "an away batch that grows while the receipt is wedged never re-sends accepted events" +} + +test_away_unusable_bookkeeping_never_sends_to_the_phone() { + local home state tg injected + home=$(make_home away-bad-bookkeeping) + state="$home/state" + tg=$(make_fake_tg "$home") + : > "$state/.afk" + injected="$home/injected.log" + printf 'blocked: needs a decision\n' > "$state/.subsuper-escalations" + printf 'not-a-batch record here\n' > "$state/.subsuper-escalations.since" + + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" > "$injected"; return 0; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" escalate_flush "$state" + ) || fail "unreadable bookkeeping must still complete the in-session flush" + + assert_absent "$home/tg-sent.log" \ + "unreadable away bookkeeping must never risk a duplicate phone alert" + assert_grep 'blocked: needs a decision' "$injected" \ + "the visible in-session fallback must carry every buffered event" + assert_grep 'ledger is unreadable' "$injected" \ + "the fallback must say why the phone was not contacted" + pass "unusable away bookkeeping fails closed to the visible in-session fallback" +} + +# The ledger reserves lines BEFORE the network call, so a reservation alone never +# proves the captain was contacted. A restart must resolve it from the durable +# delivery evidence: an unresolved attempt stays honestly uncertain and is never +# re-sent, while a reservation with no evidence at all is released so those lines +# can still reach the phone. +test_away_reserved_lines_resolve_from_evidence_not_from_the_claim() { + local home state tg injected did + home=$(make_home away-reserved-recovery) + state="$home/state" + tg=$(make_fake_tg "$home") + : > "$state/.afk" + injected="$home/injected.log" + + # (a) reserved with an unresolved `attempting` record: uncertain, never re-sent. + printf 'blocked: mid-send crash\n' > "$state/.subsuper-escalations" + did='1700000030-abcdef0123456789-0-0123456789abcdef' + mkdir -p "$state/tg-away-delivery" + chmod 0700 "$state/tg-away-delivery" + printf 'attempting 1700000030\n' > "$state/tg-away-delivery/$did.status" + chmod 0600 "$state/tg-away-delivery/$did.status" + printf '1700000030-abcdef0123456789 1 0 0 0 0 %s\n' "$did" > "$state/.subsuper-escalations.since" + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" > "$injected"; return 0; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" escalate_flush "$state" + ) || fail "an uncertain reservation must still complete the in-session flush" + assert_absent "$home/tg-sent.log" \ + "a reserved-but-unconfirmed line must never be re-sent to the phone" + assert_grep 'Telegram delivery uncertain' "$injected" \ + "an unconfirmed reservation must never be reported as accepted" + assert_grep 'blocked: mid-send crash' "$injected" \ + "the uncertain event must still reach the visible in-session fallback" + + # (a2) reserved with durable `accepted` evidence: the crash landed between the + # send and the ledger commit, so recovery confirms it from the evidence and never + # sends again. + printf 'blocked: crashed after acceptance\n' > "$state/.subsuper-escalations" + did='1700000035-abcdef0123456789-0-fedcba9876543210' + printf 'accepted 1700000035\n' > "$state/tg-away-delivery/$did.status" + chmod 0600 "$state/tg-away-delivery/$did.status" + printf '1700000035-abcdef0123456789 1 0 0 0 0 %s\n' "$did" > "$state/.subsuper-escalations.since" + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" > "$injected"; return 0; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" escalate_flush "$state" + ) || fail "a recoverable acceptance must complete the away flush" + assert_absent "$home/tg-sent.log" \ + "a line with durable accepted evidence must never be sent again" + assert_grep 'Telegram accepted away-mode alert' "$injected" \ + "durable accepted evidence must be reported as accepted" + + # (b) reserved with no evidence at all: the claim is released and the line is + # offered to the phone. + printf 'blocked: crashed before the send\n' > "$state/.subsuper-escalations" + printf '1700000040-fedcba9876543210 1 0 0 0 0 none\n' > "$state/.subsuper-escalations.since" + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" > "$injected"; return 0; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" escalate_flush "$state" + ) || fail "a released reservation must complete the away flush" + assert_grep 'crashed before the send' "$home/tg-sent.log" \ + "a reservation with no delivery evidence must be released back to the phone" + assert_grep 'Telegram accepted away-mode alert' "$injected" \ + "the re-offered event must report its real accepted outcome" + pass "reserved away lines resolve from durable evidence, never from the claim alone" +} + +# Turning Telegram mode off mid-batch must not replay into captain chat what the +# captain already received on his phone. +test_away_off_mid_batch_never_repeats_delivered_events() { + local home state tg injected + home=$(make_home away-off-mid-batch) + state="$home/state" + tg=$(make_fake_tg "$home") + : > "$state/.afk" + injected="$home/injected.log" + escalate_add "$state" 'blocked: already on the phone' + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { return 1; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" escalate_flush "$state" + ) && fail "a wedged receipt must leave escalate_flush unconfirmed" + assert_grep 'already on the phone' "$home/tg-sent.log" "the first event must reach the phone" + + # Telegram mode is switched off, then a second event appends. + rm "$home/config/telegram-mode" + escalate_add "$state" 'blocked: after the opt-out' + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" > "$injected"; return 0; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" escalate_flush "$state" + ) || fail "an opted-out away flush must still deliver in session" + assert_grep 'after the opt-out' "$injected" \ + "the undelivered event must reach captain chat once Telegram mode is off" + assert_no_grep 'already on the phone' "$injected" \ + "an event the captain already received on his phone must not be replayed in chat" + [ "$(grep -c '^---$' "$home/tg-sent.log")" -eq 1 ] \ + || fail "an opted-out flush must not contact the phone again" + pass "an opt-out mid-batch never repeats phone-delivered events in captain chat" +} + +test_away_lifecycle_retires_tg_working_records() { + local home state + home=$(make_home away-retirement) + state="$home/state" + mkdir -p "$state/tg-away-digest" "$state/tg-away-delivery" + printf 'stale persisted 300s: fm-task-1\n' > "$state/tg-away-digest/abc.items" + printf 'text' > "$state/tg-away-delivery/.spool.zzz" + printf 'accepted 1700000000\n' > "$state/tg-away-delivery/abc.status" + : > "$state/.subsuper-escalations.since.tmpXYZ" + + FM_HOME="$home" FM_STATE_OVERRIDE="$state" \ + bash -c '. "$1"; fm_afk_clear_stale_artifacts "$2"' _ "$ROOT/bin/fm-afk-start.sh" "$state" \ + || fail "the away-start clear must succeed" + + assert_absent "$state/tg-away-digest" \ + "the private away digest must retire with the away session" + assert_absent "$state/tg-away-delivery/.spool.zzz" \ + "a leaked outbound spool must retire with the away session" + assert_absent "$state/.subsuper-escalations.since.tmpXYZ" \ + "a leaked sidecar temp must retire with the away session" + [ -f "$state/tg-away-delivery/abc.status" ] \ + || fail "durable text-free delivery evidence must outlive the away session" + pass "the away lifecycle retires the Telegram working records and keeps the evidence" +} + +# --- the immutable versioned batch store ----------------------------------- +# +# Recovery is one mechanism only: the complete owned unit is published as an +# immutable version directory and reaches the live paths through ONE atomic +# active-pointer switch, performed only while no daemon is live. Nothing is ever +# merged file-by-file into the live unit or into a published version. + +seed_live_unit() { # : a two-line batch, one line already digested + local state=$1 buf="$1/.subsuper-escalations" + printf 'blocked: already digested\nblocked: still owed to captain chat\n' > "$buf" + printf '1700000000-abc 2 1 1 0 0 1700000000-abc-a0-0-d1\n' > "$buf.since" + printf 'fm away-mode inject WEDGED: 900s undelivered\n' > "$state/.subsuper-inject-wedged" + (umask 077; mkdir -p "$state/tg-away-digest") + printf 'blocked: already digested\n' > "$state/tg-away-digest/1700000000-abc-a0-0-d1.items" + chmod 600 "$state/tg-away-digest/1700000000-abc-a0-0-d1.items" +} + +# Publication is all-or-nothing: the version carries the WHOLE unit plus the +# manifest it was validated against and the completion marker written last, and +# a single rename makes it visible - so no staging directory outlives it and the +# live unit it was copied from is never disturbed. +test_away_version_publish_is_complete_and_atomic() { + local home state buf version vdir staging + home=$(make_home away-version-publish) + state="$home/state" + buf="$state/.subsuper-escalations" + seed_live_unit "$state" + + version=$(away_ledger_version_publish "$state" "$buf" "$state/.subsuper-inject-wedged") \ + || fail "publishing the live unit as an immutable version must succeed" + case "$version" in + v.*) ;; + *) fail "a published version must carry this owner's minted name: $version" ;; + esac + vdir="$state/tg-away-versions/$version" + assert_present "$vdir/.complete" "a published version must carry its completion marker" + assert_present "$vdir/manifest" "a published version must carry the manifest it was validated against" + [ "$(cat "$vdir/escalations")" = "$(cat "$buf")" ] \ + || fail "a published version must hold the escalation buffer verbatim" + [ "$(cat "$vdir/escalations.since")" = "$(cat "$buf.since")" ] \ + || fail "a published version must hold the ledger sidecar verbatim" + assert_present "$vdir/wedge" "a published version must hold the wedge marker" + assert_present "$vdir/tg-away-digest/1700000000-abc-a0-0-d1.items" \ + "a published version must hold the private digest directory" + staging=$(printf '%s' "$state"/tg-away-versions/.staging.*) + [ ! -d "$staging" ] || fail "publication must leave no staging directory behind" + assert_present "$buf" "publishing must never disturb the live unit it copied" + [ "$(away_ledger_version_names "$state")" = "$version" ] \ + || fail "the published version must be the one complete version listed" + assert_absent "$state/tg-away-versions/active" \ + "publishing must never switch the active pointer by itself" + pass "a batch version is published complete, validated, and atomically visible" +} + +# Crash safety: only the rename publishes, so an interrupted build can leave a +# staging directory or (defensively) a marker-less version directory - and +# neither may ever be listed, activated, or reach the live paths. Both are swept. +test_away_version_incomplete_publication_is_never_visible() { + local home state buf rc=0 + home=$(make_home away-version-incomplete) + state="$home/state" + buf="$state/.subsuper-escalations" + mkdir -p "$state/tg-away-versions/v.1700000000-broken" \ + "$state/tg-away-versions/.staging.zzzzzz" + printf 'blocked: never validated\n' > "$state/tg-away-versions/v.1700000000-broken/escalations" + printf 'blocked: never published\n' > "$state/tg-away-versions/.staging.zzzzzz/escalations" + + [ -z "$(away_ledger_version_names "$state")" ] \ + || fail "an incomplete version must never be listed" + away_ledger_version_activate "$state" "$buf" "$state/.subsuper-inject-wedged" \ + v.1700000000-broken || rc=$? + expect_code 1 "$rc" "activating an incomplete version must be refused" + assert_absent "$buf" "an incomplete version must never reach the live paths" + assert_absent "$state/tg-away-versions/active" \ + "a refused activation must never leave a pointer behind" + + away_ledger_version_gc "$state" || fail "the sweep must succeed" + assert_absent "$state/tg-away-versions/v.1700000000-broken" \ + "an incomplete version directory must be swept" + assert_absent "$state/tg-away-versions/.staging.zzzzzz" \ + "an abandoned staging build must be swept" + pass "an incomplete version publication is never visible, never activated, and swept" +} + +# The daemon is the sole writer of the live buffer and its sidecar with no shared +# lock, so a pointer switch under a live daemon would overwrite a freshly +# appended escalation. The owner itself refuses (rc 3) rather than trusting the +# caller, and leaves every version and the live unit untouched. +test_away_version_switch_is_refused_while_a_daemon_is_live() { + local home state buf version rc=0 + home=$(make_home away-version-live-daemon) + state="$home/state" + buf="$state/.subsuper-escalations" + seed_live_unit "$state" + version=$(away_ledger_version_publish "$state" "$buf" "$state/.subsuper-inject-wedged") \ + || fail "publishing must succeed" + rm -f "$buf" "$buf.since" "$state/.subsuper-inject-wedged" + mkdir -p "$state/.supervise-daemon.lock" + printf '%s\n' "$$" > "$state/.supervise-daemon.lock/pid" + + away_ledger_version_activate "$state" "$buf" "$state/.subsuper-inject-wedged" "$version" || rc=$? + expect_code 3 "$rc" "a pointer switch under a live daemon must be refused" + assert_absent "$state/tg-away-versions/active" \ + "a refused switch must not commit the pointer" + assert_absent "$buf" "a refused switch must not install anything live" + assert_present "$state/tg-away-versions/$version/escalations" \ + "a refused switch must leave the version untouched" + + # A switch committed before the daemon came up is not replayed under it either. + printf '%s\n' "$version" > "$state/tg-away-versions/active" + rc=0 + away_ledger_version_apply_pending "$state" "$buf" "$state/.subsuper-inject-wedged" || rc=$? + expect_code 3 "$rc" "replaying a pending switch under a live daemon must be refused" + assert_absent "$buf" "a refused replay must not install anything live" + pass "no active-pointer switch is ever performed while a daemon is live" +} + +# A switch installs ONE version's whole unit, dropping whatever is live first, so +# a version's sidecar counts can never end up paired with a foreign buffer. The +# pointer write is the commit point, so an interrupted materialisation is replayed +# to completion rather than left half-installed. +test_away_version_switch_installs_the_whole_unit_and_replays() { + local home state buf version mode + home=$(make_home away-version-switch) + state="$home/state" + buf="$state/.subsuper-escalations" + seed_live_unit "$state" + version=$(away_ledger_version_publish "$state" "$buf" "$state/.subsuper-inject-wedged") \ + || fail "publishing must succeed" + # A failed attempt's leftovers: a foreign buffer, no sidecar, no digests. + printf 'blocked: foreign line from the failed attempt\n' > "$buf" + rm -f "$buf.since" "$state/.subsuper-inject-wedged" + rm -rf "$state/tg-away-digest" + + away_ledger_version_activate "$state" "$buf" "$state/.subsuper-inject-wedged" "$version" \ + || fail "a switch with no daemon running must succeed" + assert_no_grep 'foreign line from the failed attempt' "$buf" \ + "a switch must replace the live buffer wholesale, never merge into it" + [ "$(cat "$buf")" = "$(cat "$state/tg-away-versions/$version/escalations")" ] \ + || fail "the live buffer must hold exactly the activated version's buffer" + assert_present "$buf.since" "a switch must install the version's own sidecar" + assert_present "$state/.subsuper-inject-wedged" "a switch must install the version's wedge marker" + assert_present "$state/tg-away-digest/1700000000-abc-a0-0-d1.items" \ + "a switch must install the version's private digests" + mode=$(path_mode "$state/tg-away-digest") + [ "$mode" = 700 ] || fail "the installed digest directory must stay private 0700 (got: $mode)" + [ "$(cat "$state/tg-away-versions/active")" = "$version" ] \ + || fail "the active pointer must name the switched-to version" + [ "$(cat "$state/tg-away-versions/active.applied")" = "$version" ] \ + || fail "a completed switch must record the version it materialised" + [ "$(away_ledger_fold_versions "$state" escalations)" \ + = "$(away_ledger_unaccounted_lines "$buf")" ] \ + || fail "a version whose unit is live must fold to exactly the live text, so a caller's per-kind dedupe collapses it" + + # Crash between the pointer commit and materialisation: replay converges. + rm -f "$state/tg-away-versions/active.applied" "$buf" "$buf.since" + away_ledger_version_apply_pending "$state" "$buf" "$state/.subsuper-inject-wedged" \ + || fail "an interrupted switch must be replayable" + [ "$(cat "$buf")" = "$(cat "$state/tg-away-versions/$version/escalations")" ] \ + || fail "the replay must reinstall the pointed version's whole unit" + [ "$(cat "$state/tg-away-versions/active.applied")" = "$version" ] \ + || fail "the replay must record the materialised version" + pass "an active-pointer switch installs one whole version and replays after a crash" +} + +# A retained version reaches the return catch-up exactly once: only the lines +# past that version's OWN accounted count, and only digest items whose delivery +# id is not already live. Reading never mutates the version. +test_away_fold_versions_emits_each_escalation_once() { + local home state buf vdir out + home=$(make_home away-fold-versions) + state="$home/state" + buf="$state/.subsuper-escalations" + vdir="$state/tg-away-versions/v.1700000000-retained" + mkdir -p "$vdir/tg-away-digest" + printf 'blocked: already digested\nblocked: still owed to captain chat\n' > "$vdir/escalations" + printf '1700000000-abc 2 1 1 0 0 1700000000-abc-a0-0-d1\n' > "$vdir/escalations.since" + printf 'fm away-mode inject WEDGED: 900s undelivered\n' > "$vdir/wedge" + printf 'blocked: already digested\n' > "$vdir/tg-away-digest/1700000000-abc-a0-0-d1.items" + printf 'blocked: digested but never live\n' > "$vdir/tg-away-digest/1700000000-abc-a0-1-d2.items" + printf escalations > "$vdir/manifest" + : > "$vdir/.complete" + (umask 077; mkdir -p "$state/tg-away-digest") + printf 'blocked: already digested\n' > "$state/tg-away-digest/1700000000-abc-a0-0-d1.items" + + out=$(away_ledger_fold_versions "$state" escalations) \ + || fail "folding a retained version's escalations must succeed" + assert_contains "$out" 'blocked: still owed to captain chat' \ + "a retained version's unaccounted line must reach the catch-up" + [ "$(printf '%s\n' "$out" | grep -c 'blocked: already digested')" -eq 0 ] \ + || fail "a line the version's own digest already accounted for must never be folded as owed" + out=$(away_ledger_fold_versions "$state" wedges) \ + || fail "folding a retained version's wedge marker must succeed" + assert_contains "$out" 'fm away-mode inject WEDGED: 900s undelivered' \ + "a retained version's wedge marker must reach the catch-up" + out=$(away_ledger_fold_versions "$state" digests) \ + || fail "folding a retained version's digests must succeed" + assert_contains "$out" 'blocked: digested but never live' \ + "a retained version's digest item that is not live must reach the catch-up" + [ "$(printf '%s\n' "$out" | grep -c 'blocked: already digested')" -eq 0 ] \ + || fail "a digest item whose delivery id is already live must never be folded again" + away_ledger_fold_versions "$state" bogus 2>/dev/null \ + && fail "an unknown fold kind must be refused" + assert_present "$vdir/.complete" "folding must never mutate the version it read" + pass "a retained version folds each logical escalation exactly once" +} + +# The mechanism this replaced merged single files into live paths and adopted a +# backup's artifacts under a running daemon. No entry point for that may survive +# anywhere in bin/, or the class of defect comes back with it. +test_away_no_adoption_or_merge_apis_remain() { + local hits + hits=$(grep -rn -E 'away_ledger_(reclaim_backups|fold_retained_backups|snapshot|restore)|afk-launch-backup|digests-only' \ + "$ROOT/bin" 2>/dev/null || true) + [ -z "$hits" ] \ + || fail "the in-place adoption/merge recovery mechanism must be gone: $hits" + pass "no adoption or per-file merge API survives in bin/" +} + +# fn_body : the source text of one shell function, so a +# structural assertion can be made about that function alone rather than about a +# whole file. +fn_body() { + awk -v fn="$2() {" 'index($0, fn) == 1 { f = 1 } f { print } f && /^}$/ { exit }' "$1" +} + +# Structural, not behavioural: EVERY ledger transition must go through the version +# transaction, and no transition may write a legacy live path as its source of +# truth. Behaviour tests below prove the transaction works; this one proves nothing +# quietly bypasses it. +test_away_every_transition_goes_through_the_version_transaction() { + local lib daemon body fn + lib="$ROOT/bin/fm-away-ledger-lib.sh" + daemon="$ROOT/bin/fm-supervise-daemon.sh" + + for fn in away_ledger_write away_ledger_append away_ledger_truncate \ + away_ledger_wedge_record away_ledger_wedge_clear away_ledger_digest_record; do + body=$(fn_body "$lib" "$fn") + [ -n "$body" ] || fail "$fn must exist as an owner transition" + assert_contains "$body" 'away_ledger_transact' \ + "$fn must commit a successor version through away_ledger_transact" + done + + # The reservation, acceptance and retry-state transitions reach the transaction + # through the single record-write path. + for fn in away_ledger_reserve away_ledger_confirm away_ledger_release \ + away_ledger_transition; do + body=$(fn_body "$lib" "$fn") + [ -n "$body" ] || fail "$fn must exist as an owner transition" + case "$body" in + *away_ledger_write*|*away_ledger_transition*) ;; + *) fail "$fn must route its record change through away_ledger_write" ;; + esac + done + + body=$(fn_body "$daemon" escalate_add) + assert_contains "$body" 'away_ledger_append' \ + "the daemon's escalation append must be a version transition" + # shellcheck disable=SC2016 # Literal shell syntax is the asserted pattern. + assert_not_contains "$body" '>> "$buf"' \ + "the daemon must not append straight into the live escalation buffer" + + body=$(fn_body "$daemon" escalate_flush) + assert_contains "$body" 'away_ledger_truncate' \ + "the in-session truncation must be a version transition" + # shellcheck disable=SC2016 # Literal shell syntax is the asserted pattern. + assert_not_contains "$body" ': > "$buf"' \ + "the daemon must not truncate the live escalation buffer directly" + + body=$(fn_body "$daemon" inject_wedge_alarm) + assert_contains "$body" 'away_ledger_wedge_record' \ + "the wedge evidence must be a version transition" + # shellcheck disable=SC2016 # Literal shell syntax is the asserted pattern. + assert_not_contains "$body" '> "$marker"' \ + "the daemon must not write the live wedge marker directly" + + # shellcheck disable=SC2016 # Literal shell syntax is the asserted pattern. + body=$(grep -nE '>>?[[:space:]]*"\$state/\.subsuper-(escalations|inject-wedged|chat-delivery)' "$daemon" || true) + [ -z "$body" ] || fail "no live away artifact may be written outside the transaction: $body" + pass "every ledger transition commits a successor version and no live write remains" +} + +# The append transition: the line is durable in a published version, the active +# pointer names it, and the live buffer is that version's projection. +test_away_append_transition_publishes_and_projects() { + local home state buf name + home=$(make_home away-append-transition) + state="$home/state" + buf="$state/.subsuper-escalations" + : > "$state/.afk" + + escalate_add "$state" 'blocked: first event' || fail "the append transition must succeed" + escalate_add "$state" 'blocked: second event' || fail "the second append must succeed" + + name=$(cat "$state/tg-away-versions/active" 2>/dev/null) + [ -n "$name" ] || fail "an append must leave an active version" + [ "$(cat "$state/tg-away-versions/active.applied")" = "$name" ] \ + || fail "an applied append must record the version it projected" + [ "$(cat "$buf")" = "$(cat "$state/tg-away-versions/$name/escalations")" ] \ + || fail "the live buffer must be the active version's projection" + assert_grep 'blocked: second event' "$state/tg-away-versions/$name/escalations" \ + "the appended line must be durable in the published version" + [ "$(away_ledger_version_names "$state" | wc -l | tr -d ' ')" = 1 ] \ + || fail "a superseded predecessor must be retired by its successor" + pass "the append transition publishes a successor version and projects it live" +} + +# The reservation, acceptance, digest and accounted transitions each publish their +# own successor version, and the digest item and the count that accounts for it +# land in the SAME version. +test_away_record_and_digest_transitions_publish_versions() { + local home state buf name rec + home=$(make_home away-record-transition) + state="$home/state" + buf="$state/.subsuper-escalations" + : > "$state/.afk" + escalate_add "$state" 'blocked: needs the captain' || fail "the append must succeed" + + away_ledger_reserve "$buf" 1 d-1 || fail "the reservation transition must succeed" + name=$(cat "$state/tg-away-versions/active") + rec=$(away_ledger_peek "$state/tg-away-versions/$name/escalations") + case "$rec" in + *' 1 0 0 0 0 d-1') ;; + *) fail "the reservation must be durable in its own successor version: $rec" ;; + esac + + away_ledger_confirm "$buf" 0 1 d-1 || fail "the acceptance transition must succeed" + away_ledger_digest_record "$state" "$buf" d-1 0 1 \ + || fail "the digest transition must succeed" + name=$(cat "$state/tg-away-versions/active") + assert_grep 'blocked: needs the captain' \ + "$state/tg-away-versions/$name/tg-away-digest/d-1.items" \ + "the digest item must be durable in the successor version" + rec=$(away_ledger_peek "$state/tg-away-versions/$name/escalations") + case "$rec" in + *' 1 1 1 0 0 d-1') ;; + *) fail "the accounted count must land in the same version as its digest: $rec" ;; + esac + assert_grep 'blocked: needs the captain' "$state/tg-away-digest/d-1.items" \ + "the live digest directory must be the version's projection" + pass "record and digest transitions each publish a successor version" +} + +# The wedge evidence and the in-session truncation are transitions too: the +# truncation empties the buffer and drops the record and the wedge marker as ONE +# version, so no half-retired unit is ever published or projected. +test_away_wedge_and_truncate_transitions_publish_versions() { + local home state buf name + home=$(make_home away-wedge-transition) + state="$home/state" + buf="$state/.subsuper-escalations" + : > "$state/.afk" + escalate_add "$state" 'blocked: wedged event' || fail "the append must succeed" + + printf 'fm away-mode inject WEDGED: 900s undelivered\n' \ + | away_ledger_wedge_record "$state" || fail "the wedge transition must succeed" + name=$(cat "$state/tg-away-versions/active") + assert_grep 'WEDGED: 900s' "$state/tg-away-versions/$name/wedge" \ + "the wedge evidence must be durable in the successor version" + assert_grep 'WEDGED: 900s' "$state/.subsuper-inject-wedged" \ + "the live wedge marker must be the version's projection" + + away_ledger_truncate "$buf" || fail "the truncate transition must succeed" + name=$(cat "$state/tg-away-versions/active") + [ ! -s "$buf" ] || fail "the truncation must empty the live buffer" + assert_absent "$buf.since" "the truncation must drop the live record" + assert_absent "$state/.subsuper-inject-wedged" \ + "the truncation must drop the live wedge marker" + assert_absent "$state/tg-away-versions/$name/escalations.since" \ + "the successor version must hold the retired record state" + pass "the wedge and truncate transitions publish successor versions" +} + +# An unreadable predecessor record must be quarantined as immutable evidence, not +# treated as a publication failure: away-mode entry can never be wedged by +# bookkeeping nobody can parse, and no owed line is dropped. +test_away_unparseable_record_is_quarantined_not_fatal() { + local home state buf version out + home=$(make_home away-unparseable-record) + state="$home/state" + buf="$state/.subsuper-escalations" + printf 'blocked: owed, with an unreadable record\n' > "$buf" + : > "$buf.since" + + version=$(away_ledger_entry_capture "$state" "$buf" "$state/.subsuper-inject-wedged") \ + || fail "an unparseable record must never wedge an away entry" + assert_grep 'owed, with an unreadable record' \ + "$state/tg-away-versions/$version/escalations" \ + "the quarantined version must still hold every owed line" + [ -e "$state/tg-away-versions/$version/escalations.since" ] \ + || fail "the malformed record itself must be quarantined as evidence" + [ ! -s "$buf.since" ] || fail "the live record must be left exactly as found" + + away_ledger_retire_batch "$state" "$buf" "$state/.subsuper-inject-wedged" \ + || fail "the entry clear must succeed" + out=$(away_ledger_fold_versions "$state" escalations) \ + || fail "folding a quarantined version must succeed" + assert_contains "$out" 'owed, with an unreadable record' \ + "an unreadable record must make the fold over-report, never drop a line" + pass "an unparseable predecessor record is quarantined instead of wedging entry" +} + +# A pointer left aiming at a version that is gone would otherwise refuse every +# later apply, and so every away entry, for good. +test_away_gc_clears_a_dangling_active_pointer() { + local home state buf + home=$(make_home away-dangling-pointer) + state="$home/state" + buf="$state/.subsuper-escalations" + (umask 077; mkdir -p "$state/tg-away-versions") + printf 'v.1700000000-gone\n' > "$state/tg-away-versions/active" + printf 'v.1700000000-gone\n' > "$state/tg-away-versions/active.applied" + + away_ledger_version_gc "$state" || fail "the sweep must succeed" + assert_absent "$state/tg-away-versions/active" \ + "a pointer naming a version that is gone must be cleared" + assert_absent "$state/tg-away-versions/active.applied" \ + "an applied pointer naming a version that is gone must be cleared" + away_ledger_version_apply_pending "$state" "$buf" "$state/.subsuper-inject-wedged" \ + || fail "away entry must not stay wedged once the dangling pointer is cleared" + pass "the sweep clears a dangling pointer instead of wedging every away entry" +} + +# Mutual exclusion is what stops a publish interleaving with a materialise, so a +# stale-lock reclaim must never remove a lock a NEW owner has since taken - the +# double-remove race two reclaimers that saw the same dead holder would hit. +test_away_store_lock_steal_never_removes_a_new_owner() { + local home state dir lock dead + home=$(make_home away-lock-steal) + state="$home/state" + dir="$state/tg-away-versions" + (umask 077; mkdir -p "$dir") + lock="$dir/.owner.lock" + + # A holder that is provably gone is reclaimed. + dead=$(bash -c 'echo $$') + while kill -0 "$dead" 2>/dev/null; do dead=$((dead + 1)); done + mkdir "$lock" + printf '%s\n' "$dead" > "$lock/pid" + _away_ledger_lock_steal "$lock" "$dead" + assert_absent "$lock" "a lock whose holder is provably gone must be reclaimed" + + # The loser of the steal race re-reads the holder and must leave the new owner + # alone, even though it decided to reclaim while observing the dead one. + mkdir "$lock" + printf '%s\n' "$$" > "$lock/pid" + _away_ledger_lock_steal "$lock" "$dead" + assert_present "$lock" "a reclaim must never remove a lock a new owner holds" + [ "$(cat "$lock/pid")" = "$$" ] || fail "the new owner's lock identity was replaced" + assert_absent "$lock.steal" "the steal lock must not be left behind" + rm -rf "$lock" + + away_ledger_lock_acquire "$dir" || fail "acquiring the store lock must succeed" + [ "$(cat "$lock/pid")" = "$$" ] || fail "the acquired lock must record its holder" + away_ledger_lock_release "$dir" + assert_absent "$lock" "releasing the store lock must remove it" + pass "a stale store-lock reclaim can never hand the store to two owners" +} + +# A refused append must leave every suppressor untouched. The seen marker is what +# the heartbeat catch-all checks, so recording it for an escalation that reached no +# buffer, no version and no digest would suppress the only re-derivation path and +# lose a captain-relevant event outright. +test_away_refused_append_keeps_the_wake_re_derivable() { + local home state seen + home=$(make_home away-refused-append) + state="$home/state" + : > "$state/.afk" + printf 'blocked [key=needs-captain]: the captain must decide\n' > "$state/t1.status" + seen="$state/.subsuper-seen-status-t1" + + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + away_ledger_append() { return 1; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + log() { :; } + handle_wake "signal: $state/t1.status" "$state" + ) >/dev/null 2>&1 + assert_absent "$seen" \ + "a refused append must not record the seen suppressor the catch-all scan checks" + [ ! -s "$state/.subsuper-escalations" ] \ + || fail "a refused append must leave nothing buffered" + + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + ( log() { :; }; handle_wake "signal: $state/t1.status" "$state" ) >/dev/null 2>&1 + assert_grep 'the captain must decide' "$state/.subsuper-escalations" \ + "the retried wake must buffer the escalation the refused append dropped" + assert_present "$seen" "a committed append must record the seen suppressor" + pass "a refused append leaves the wake re-derivable instead of suppressing it" +} + +# Once the in-session submit is confirmed, a failure in the bookkeeping that +# should have followed must never become a second delivery of the same batch into +# captain chat. +test_away_confirmed_inject_is_never_repeated_after_a_failed_truncate() { + local home state buf injected count + home=$(make_home away-confirmed-inject) + state="$home/state" + buf="$state/.subsuper-escalations" + injected="$home/injected.log" + : > "$state/.afk" + : > "$injected" + escalate_add "$state" 'blocked: one event the captain already saw' \ + || fail "the append must succeed" + + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" >> "$injected"; return 0; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + away_ledger_truncate() { return 1; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + log() { :; } + escalate_flush "$state" + ) && fail "a failed truncation must still be reported as a flush failure" + count=$(grep -c 'one event the captain already saw' "$injected") + [ "$count" -eq 1 ] || fail "the first flush must inject exactly once (got $count)" + [ "$(away_ledger_chat_delivered "$state")" = 1 ] \ + || fail "a confirmed submit must be recorded in the unit before the truncation" + + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" >> "$injected"; return 0; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + log() { :; } + escalate_flush "$state" + ) || fail "the retry must settle the batch once the truncation can land" + count=$(grep -c 'one event the captain already saw' "$injected") + [ "$count" -eq 1 ] \ + || fail "a confirmed batch must never be re-injected into captain chat (got $count)" + [ ! -s "$buf" ] || fail "the retry must complete the outstanding truncation" + assert_absent "$state/.subsuper-chat-delivery" \ + "a retired batch must not leave its in-session delivery state behind" + pass "a confirmed in-session delivery is never repeated after a failed truncation" +} + +# A batch only ever grows, so the confirmed in-session delivery is a PREFIX, not +# an exact line count: a line appended between a confirmed submit and the +# truncation that should have followed must not drag the already-delivered lines +# back into captain chat. +test_away_confirmed_chat_prefix_survives_a_growing_batch() { + local home state buf injected count + home=$(make_home away-confirmed-prefix) + state="$home/state" + buf="$state/.subsuper-escalations" + injected="$home/injected.log" + : > "$state/.afk" + : > "$injected" + escalate_add "$state" 'blocked: first event the captain already saw' \ + || fail "the first append must succeed" + + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" >> "$injected"; return 0; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + away_ledger_truncate() { return 1; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + log() { :; } + escalate_flush "$state" + ) && fail "a failed truncation must still be reported as a flush failure" + + escalate_add "$state" 'blocked: second event arriving after the confirmed submit' \ + || fail "the append onto the confirmed batch must succeed" + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" >> "$injected"; return 0; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + log() { :; } + escalate_flush "$state" + ) || fail "the grown batch must still flush" + + count=$(grep -c 'first event the captain already saw' "$injected") + [ "$count" -eq 1 ] \ + || fail "a confirmed prefix must not be re-injected when the batch grows (got $count)" + grep -F 'second event arriving after the confirmed submit' "$injected" >/dev/null \ + || fail "the line appended past the confirmed prefix must still reach captain chat" + [ ! -s "$buf" ] || fail "the grown batch must be retired once its flush settles" + pass "a confirmed in-session prefix is never repeated when the batch grows" +} + +# Opening an attempt states which suffix is in flight; it must never lower the +# confirmed prefix. Otherwise a failed inject on a grown batch would erase the +# proof that the earlier lines already reached captain chat and re-deliver them. +test_away_failed_inject_never_lowers_the_confirmed_chat_prefix() { + local home state buf injected count + home=$(make_home away-attempt-prefix) + state="$home/state" + buf="$state/.subsuper-escalations" + injected="$home/injected.log" + : > "$state/.afk" + : > "$injected" + escalate_add "$state" 'blocked: first event the captain already saw' \ + || fail "the first append must succeed" + + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" >> "$injected"; return 0; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + away_ledger_truncate() { return 1; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + log() { :; } + escalate_flush "$state" + ) && fail "a failed truncation must still be reported as a flush failure" + + escalate_add "$state" 'blocked: second event arriving after the confirmed submit' \ + || fail "the append onto the confirmed batch must succeed" + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { return 1; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + log() { :; } + escalate_flush "$state" + ) && fail "a refused inject must be reported as a flush failure" + [ "$(away_ledger_chat_delivered "$state")" = 1 ] \ + || fail "an attempt that failed to submit must not lower the confirmed prefix" + + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" >> "$injected"; return 0; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + log() { :; } + escalate_flush "$state" + ) || fail "the retry after a refused inject must settle the batch" + count=$(grep -c 'first event the captain already saw' "$injected") + [ "$count" -eq 1 ] \ + || fail "a refused inject must not re-deliver the confirmed prefix (got $count)" + grep -F 'second event arriving after the confirmed submit' "$injected" >/dev/null \ + || fail "the unconfirmed suffix must still reach captain chat" + pass "a refused inject never lowers the confirmed in-session prefix" +} + +# The receipt names its own batch and lives beside the ledger, not inside it, so +# a sidecar corrupted after a confirmed submit still fails OPEN (the unconfirmed +# lines are surfaced) without repeating the prefix captain chat already has. +test_away_corrupt_ledger_keeps_the_confirmed_chat_prefix() { + local home state buf injected count + home=$(make_home away-corrupt-after-chat) + state="$home/state" + buf="$state/.subsuper-escalations" + injected="$home/injected.log" + : > "$state/.afk" + : > "$injected" + escalate_add "$state" 'blocked: first event the captain already saw' \ + || fail "the first append must succeed" + + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" >> "$injected"; return 0; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + away_ledger_truncate() { return 1; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + log() { :; } + escalate_flush "$state" + ) && fail "a failed truncation must still be reported as a flush failure" + + escalate_add "$state" 'blocked: second event after the ledger went bad' \ + || fail "the append onto the confirmed batch must succeed" + printf 'garbage\n' > "$buf.since" + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" >> "$injected"; return 0; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + log() { :; } + escalate_flush "$state" + ) || fail "an unreadable ledger must still flush to the in-session fallback" + + count=$(grep -c 'first event the captain already saw' "$injected") + [ "$count" -eq 1 ] \ + || fail "an unreadable ledger must not repeat the confirmed prefix (got $count)" + grep -F 'second event after the ledger went bad' "$injected" >/dev/null \ + || fail "an unreadable ledger must still surface the unconfirmed lines" + pass "an unreadable ledger keeps the confirmed in-session prefix out of captain chat" +} + +# A batch whose payload goes unreadable BEFORE its first in-session delivery has +# no receipt to adopt an identity from, so the store falls back to the identity of +# its own validated active version. Without that the prefix could never be opened +# and every retry would repeat the lines captain chat already has. +test_away_first_chat_delivery_under_a_corrupt_ledger_opens_a_receipt() { + local home state buf injected count + home=$(make_home away-corrupt-first-chat) + state="$home/state" + buf="$state/.subsuper-escalations" + injected="$home/injected.log" + : > "$state/.afk" + : > "$injected" + escalate_add "$state" 'blocked: first event under an unreadable ledger' \ + || fail "the append must succeed" + assert_absent "$state/.subsuper-chat-delivery" \ + "the batch must reach its first flush with no in-session receipt" + printf 'garbage\n' > "$buf.since" + + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" >> "$injected"; return 0; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + away_ledger_truncate() { return 1; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + log() { :; } + escalate_flush "$state" + ) && fail "a failed truncation must still be reported as a flush failure" + [ "$(away_ledger_chat_delivered "$state")" = 1 ] \ + || fail "an unreadable ledger must still be able to open a confirmed chat prefix" + + escalate_add "$state" 'blocked: second event under an unreadable ledger' \ + || fail "the append onto the delivered batch must succeed" + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" >> "$injected"; return 0; } + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + log() { :; } + escalate_flush "$state" + ) || fail "the retry must settle the batch" + count=$(grep -c 'first event under an unreadable ledger' "$injected") + [ "$count" -eq 1 ] \ + || fail "the prefix opened under an unreadable ledger must not be re-delivered (got $count)" + grep -F 'second event under an unreadable ledger' "$injected" >/dev/null \ + || fail "the unconfirmed suffix must still reach captain chat" + pass "a first in-session delivery under an unreadable ledger still opens a confirmed prefix" +} + +# The daemon's TERM/INT cleanup runs its final flush inside the same shell, so a +# transition must be able to nest inside a transaction this process already holds - +# and a nested release must never drop the outer transaction's lock. +test_away_store_lock_is_reentrant_for_its_own_holder() { + local home state dir buf + home=$(make_home away-lock-reentrant) + state="$home/state" + buf="$state/.subsuper-escalations" + dir="$state/tg-away-versions" + (umask 077; mkdir -p "$dir") + + away_ledger_lock_acquire "$dir" || fail "acquiring the store lock must succeed" + away_ledger_append "$buf" 'blocked: nested under the holder-owned lock' \ + || fail "a nested transition must not spin against a lock this process holds" + assert_present "$dir/.owner.lock" \ + "a nested release must not drop the outer transaction's lock" + assert_grep 'nested under the holder-owned lock' "$buf" \ + "the nested transition must still have committed and projected" + away_ledger_lock_release "$dir" + assert_absent "$dir/.owner.lock" "the outermost release must drop the lock" + pass "the store lock is reentrant for its own holder and only the outer release frees it" +} + +# The predecessor is only retired when the successor was provably BUILT from it. +# A live unit that diverged - an interrupted projection, or a writer outside this +# owner - is staged from live instead, and its predecessor stays retained for the +# return fold rather than being deleted on an unverified superset assumption. +test_away_diverged_live_unit_retains_its_predecessor() { + local home state buf first second out + home=$(make_home away-diverged-unit) + state="$home/state" + buf="$state/.subsuper-escalations" + : > "$state/.afk" + + escalate_add "$state" 'blocked: first event' || fail "the first append must succeed" + first=$(cat "$state/tg-away-versions/active") + escalate_add "$state" 'blocked: second event' || fail "the second append must succeed" + second=$(cat "$state/tg-away-versions/active") + assert_absent "$state/tg-away-versions/$first" \ + "a successor built from its predecessor must retire it" + + rm -f "$buf.since" + escalate_add "$state" 'blocked: third event' || fail "the third append must succeed" + [ "$(cat "$state/tg-away-versions/active")" != "$second" ] \ + || fail "the diverged transition must have published its own successor" + assert_present "$state/tg-away-versions/$second/escalations" \ + "a successor staged from a diverged live unit must retain its predecessor" + out=$(away_ledger_fold_versions "$state" escalations) \ + || fail "folding the retained predecessor must succeed" + assert_contains "$out" 'blocked: first event' \ + "the retained predecessor must still reach the catch-up" + pass "a predecessor is retired only when the successor was built from it" +} + +# The wedge and chat staging temps are part of the unit's temp set, so a crash +# between mktemp and its removal must not outlive a lifecycle boundary. +test_away_lifecycle_sweeps_leaked_unit_staging_temps() { + local home state + home=$(make_home away-temp-sweep) + state="$home/state" + : > "$state/.subsuper-inject-wedged.stage.zzzzzz" + : > "$state/.subsuper-chat-delivery.apply.zzzzzz" + + away_ledger_retire_working_records "$state" || fail "the sweep must succeed" + assert_absent "$state/.subsuper-inject-wedged.stage.zzzzzz" \ + "a leaked wedge staging temp must retire with the away session" + assert_absent "$state/.subsuper-chat-delivery.apply.zzzzzz" \ + "a leaked in-session delivery temp must retire with the away session" + pass "leaked unit staging temps retire with the away session" +} + +# Every identity this owner mints must be unique, and the mint runs from inside +# command substitutions where bash 3.2 restarts $RANDOM from the caller's state. +# Two mints in the same second colliding would give two batches one identity and +# two versions one directory name - and a rename onto an existing version name +# publishes the PREDECESSOR as though it were the successor, silently dropping the +# transition that was just committed. +test_away_minted_identities_do_not_collide_within_one_second() { + local home state buf version first second + home=$(make_home away-mint-unique) + state="$home/state" + buf="$state/.subsuper-escalations" + : > "$state/.afk" + + escalate_add "$state" 'blocked: first within the same second' \ + || fail "the first append must succeed" + escalate_add "$state" 'blocked: second within the same second' \ + || fail "the second append must succeed" + [ "$(wc -l < "$buf" | tr -d ' ')" -eq 2 ] \ + || fail "two appends in the same second must both survive: [$(cat "$buf")]" + version=$(cat "$state/tg-away-versions/active") + [ "$(wc -l < "$state/tg-away-versions/$version/escalations" | tr -d ' ')" -eq 2 ] \ + || fail "the active version must hold both appends" + [ -z "$(find "$state/tg-away-versions" -name '.staging.*' 2>/dev/null)" ] \ + || fail "a version must never contain a nested staging directory" + + away_ledger_truncate "$buf" || fail "the truncation must succeed" + away_ledger_open "$buf" || fail "opening a fresh batch must succeed" + first=$(cut -d ' ' -f1 < "$buf.since") + away_ledger_truncate "$buf" || fail "the second truncation must succeed" + away_ledger_open "$buf" || fail "opening a second fresh batch must succeed" + second=$(cut -d ' ' -f1 < "$buf.since") + [ "$first" != "$second" ] \ + || fail "two batch identities minted in the same second must differ ($first)" + pass "minted identities never collide within one second" +} + +# Enforcement, not behaviour: the version store has exactly one writer, and the +# migrating record read is never pointed at a version or staging directory. Both +# are the invariants the immutability of a published version rests on, so they are +# checked structurally rather than left to review. +test_away_only_the_owner_writes_the_version_store() { + local hits + hits=$(grep -rn -E '(>>?[[:space:]]*"?[^|]*|cp |mv |rm |mkdir |chmod |touch )tg-away-versions' \ + "$ROOT/bin" 2>/dev/null | grep -v '/fm-away-ledger-lib\.sh:' | grep -v '^[^:]*:[0-9]*:#' || true) + [ -z "$hits" ] \ + || fail "only the away ledger owner may write the version store: $hits" + # shellcheck disable=SC2016 # Literal shell syntax is the asserted pattern. + hits=$(grep -rn -E 'away_ledger_read "\$(vdir|staging)' "$ROOT/bin" 2>/dev/null || true) + [ -z "$hits" ] \ + || fail "the migrating record read must never be pointed at a version or staging directory: $hits" + pass "the version store has a single writer and no migrating read inside a version" +} + +# The record reader splits in two on purpose: away_ledger_peek must never write, +# because it is the reader pointed at published immutable versions, while +# away_ledger_read keeps migrating an earlier record shape in place for the LIVE +# buffer the daemon owns. +test_away_peek_never_writes_and_read_migrates_only_live() { + local home state buf before rec + home=$(make_home away-peek-pure) + state="$home/state" + buf="$state/.subsuper-escalations" + printf 'blocked: from a pre-ledger session\n' > "$buf" + printf '1700000000\n' > "$buf.since" + before=$(cat "$buf.since") + + rec=$(away_ledger_peek "$buf") + [ "$rec" != unknown ] || fail "peek must normalise a pre-ledger record rather than fail closed" + [ "$(cat "$buf.since")" = "$before" ] \ + || fail "peek must never write the record it read" + case "$rec" in + *' 0 0 0 0 0 none') ;; + *) fail "peek must report zeroed counts for a pre-ledger record: $rec" ;; + esac + + rec=$(away_ledger_read "$buf") + [ "$rec" != unknown ] || fail "the live read must migrate a pre-ledger record" + [ "$(cat "$buf.since")" != "$before" ] \ + || fail "the live read must still migrate an earlier record shape in place" + pass "the pure record read never writes, and only the live read migrates" +} + +# Migration belongs to successor construction: publishing normalises the record +# inside its own private staging copy, so the version carries a current-shape +# record and the live sidecar it was copied from is left exactly as it was. +test_away_version_publish_migrates_into_the_successor_only() { + local home state buf version vdir + home=$(make_home away-publish-migrates) + state="$home/state" + buf="$state/.subsuper-escalations" + printf 'blocked: owed since before the ledger\n' > "$buf" + printf '1700000000\n' > "$buf.since" + + version=$(away_ledger_version_publish "$state" "$buf" "$state/.subsuper-inject-wedged") \ + || fail "publishing must succeed over an earlier record shape" + vdir="$state/tg-away-versions/$version" + [ "$(cat "$buf.since")" = 1700000000 ] \ + || fail "publishing must not migrate the live sidecar it copied" + [ "$(awk '{print NF}' "$vdir/escalations.since")" = 7 ] \ + || fail "the successor version must carry a current-shape record: $(cat "$vdir/escalations.since")" + [ "$(away_ledger_peek "$vdir/escalations")" != unknown ] \ + || fail "the successor version's record must be readable without migration" + pass "an earlier record shape is migrated only into the successor version" +} + +# The fold at return is strictly read-only, including over a version that +# preserved an earlier record shape - the case a migrating read would have +# rewritten inside a published version. +test_away_fold_versions_never_mutates_a_legacy_version() { + local home state buf vdir before out + home=$(make_home away-fold-immutable) + state="$home/state" + buf="$state/.subsuper-escalations" + vdir="$state/tg-away-versions/v.1700000000-legacy" + (umask 077; mkdir -p "$vdir") + printf 'blocked: legacy shape, still owed\n' > "$vdir/escalations" + printf '1700000000\n' > "$vdir/escalations.since" + printf 'escalations\n' > "$vdir/manifest" + : > "$vdir/.complete" + before=$(cat "$vdir/escalations.since") + + out=$(away_ledger_fold_versions "$state" escalations) \ + || fail "folding a legacy-shape version must succeed" + assert_contains "$out" 'blocked: legacy shape, still owed' \ + "a legacy-shape version's owed line must still reach the catch-up" + [ "$(cat "$vdir/escalations.since")" = "$before" ] \ + || fail "the fold must never write inside a published immutable version" + pass "the return fold never mutates a published version, whatever record shape it holds" +} + +# One entry-boundary transaction owns capture-before-clear, so no away entry path +# can delete a crashed session's un-flushed lines with no surviving copy. +test_away_entry_capture_keeps_the_predecessor_before_a_clear() { + local home state buf version out + home=$(make_home away-entry-capture) + state="$home/state" + buf="$state/.subsuper-escalations" + seed_live_unit "$state" + + version=$(away_ledger_entry_capture "$state" "$buf" "$state/.subsuper-inject-wedged") \ + || fail "the entry-boundary capture must succeed" + assert_present "$buf" "capture must leave the live unit for the caller to clear" + away_ledger_retire_batch "$state" "$buf" "$state/.subsuper-inject-wedged" \ + || fail "retiring the live unit must succeed" + assert_absent "$buf" "the live unit must be gone after the entry clear" + + out=$(away_ledger_fold_versions "$state" escalations) \ + || fail "folding the captured version must succeed" + assert_contains "$out" 'blocked: still owed to captain chat' \ + "the captured version must still hold what the entry clear removed" + [ "$(away_ledger_version_names "$state")" = "$version" ] \ + || fail "the capture must leave exactly the version it published" + pass "the entry boundary captures the predecessor before any clear removes it" +} + +# Retirement is acknowledgement: it must report failure while any version +# survives, so a caller cannot close its gate over content that will be folded +# again. +test_away_versions_retire_all_refuses_while_a_version_survives() { + local home state buf version rc=0 + home=$(make_home away-retire-fault) + state="$home/state" + buf="$state/.subsuper-escalations" + seed_live_unit "$state" + version=$(away_ledger_version_publish "$state" "$buf" "$state/.subsuper-inject-wedged") \ + || fail "publishing must succeed" + chmod 500 "$state/tg-away-versions/$version" + + away_ledger_versions_retire_all "$state" || rc=$? + expect_code 1 "$rc" "retirement must report failure while a version cannot be removed" + [ "$(away_ledger_version_names "$state")" = "$version" ] \ + || fail "the unremovable version must still be listed as retained" + + chmod 700 "$state/tg-away-versions/$version" + away_ledger_versions_retire_all "$state" \ + || fail "retirement must succeed once the version can be removed" + [ -z "$(away_ledger_version_names "$state")" ] \ + || fail "acknowledged retirement must leave no version behind" + pass "whole-version retirement reports failure until the store is provably empty" +} + +# An uncertain outcome poisons the batch for the phone: confirmation can only ever +# extend a contiguous proven prefix, so a later line is never sent (which would +# risk a duplicate of the uncertain one) and never presented as delivered. +test_away_uncertain_send_stops_the_batch_from_reaching_the_phone() { + local home state tg injected + home=$(make_home away-uncertain-gap) + state="$home/state" + tg=$(make_fake_tg "$home") + : > "$state/.afk" + injected="$home/injected.log" + + escalate_add "$state" 'blocked: first event, response lost' + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { return 1; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" FAKE_TG_EXIT=1 escalate_flush "$state" + ) && fail "a wedged receipt must leave escalate_flush unconfirmed" + [ "$(grep -c '^---$' "$home/tg-sent.log")" -eq 1 ] \ + || fail "the first event must have been attempted exactly once" + ledger_field "$state" 3 0 "an uncertain send must not confirm anything" + + escalate_add "$state" 'blocked: second event after the uncertainty' + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" > "$injected"; return 0; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" escalate_flush "$state" + ) || fail "an uncertain batch must still complete the in-session flush" + + [ "$(grep -c '^---$' "$home/tg-sent.log")" -eq 1 ] \ + || fail "no event may reach the phone after an uncertain send in the same batch" + assert_grep 'Telegram delivery uncertain' "$injected" \ + "an uncertain batch must never be reported as accepted" + assert_grep 'first event, response lost' "$injected" \ + "the uncertain event must reach the visible in-session fallback" + assert_grep 'second event after the uncertainty' "$injected" \ + "the later event must reach the visible in-session fallback" + assert_no_grep 'accepted away-mode alert' "$injected" \ + "an unproven line must never be presented as delivered" + pass "an uncertain send keeps its reservation and stops the batch reaching the phone" +} + +# A client that actually ran and exited 125 (or 124) must never be treated as +# proven-local: those exit codes belong exclusively to wedge_alarm_run_bounded's +# own pre-launch guard and timeout. A client-produced 125 must stay ambiguous - +# reservation kept, attempt ordinal unchanged - so the same lines are never +# offered to the phone again under a fresh delivery id (which would duplicate +# the captain alert the client may already have sent). +test_away_client_exit_125_is_never_proven_local() { + local home state tg injected + home=$(make_home away-client-exit-125) + state="$home/state" + tg=$(make_fake_tg "$home") + : > "$state/.afk" + injected="$home/injected.log" + escalate_add "$state" 'blocked: client exited 125 after issuing the request' + + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" > "$injected"; return 1; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" FAKE_TG_EXIT=125 escalate_flush "$state" + ) && fail "a client exit 125 must leave escalate_flush unconfirmed" + [ "$(grep -c '^---$' "$home/tg-sent.log")" -eq 1 ] \ + || fail "the client must have been invoked exactly once" + ledger_field "$state" 2 1 "a client exit 125 must keep its reservation" + ledger_field "$state" 3 0 "a client exit 125 must not confirm anything" + ledger_field "$state" 5 0 "a client exit 125 must not advance the attempt ordinal" + assert_grep 'Telegram delivery uncertain' "$injected" \ + "a client exit 125 must never be reported as a proven-local failure" + + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" > "$injected"; return 0; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" escalate_flush "$state" + ) || fail "a poisoned batch must still complete the in-session flush" + [ "$(grep -c '^---$' "$home/tg-sent.log")" -eq 1 ] \ + || fail "an ambiguous exit 125 must never be retried, which would risk a duplicate phone alert" + pass "a client exit 125 stays ambiguous and is never retried as proven-local" +} + +# Every send must come through the ledger: no default arguments, no id minted +# outside the owner, no bypass of the exactly-once accounting. +test_away_delivery_refuses_untracked_sends() { + local home state tg out + home=$(make_home away-untracked) + state="$home/state" + tg=$(make_fake_tg "$home") + : > "$state/.afk" + escalate_add "$state" 'blocked: needs the ledger' + + out=$( + # shellcheck disable=SC2030,SC2031 # The exports are deliberately scoped to this subshell. + export FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live + # shellcheck disable=SC2030,SC2031 # The exports are deliberately scoped to this subshell. + export FMTG_TG_BIN="$tg" + telegram_away_deliver "$state" 'blocked: needs the ledger' + ) + [ "$out" = 'unavailable|none' ] \ + || fail "a send without ledger arguments must refuse (got: $out)" + + out=$( + # shellcheck disable=SC2030,SC2031 # The exports are deliberately scoped to this subshell. + export FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live + # shellcheck disable=SC2030,SC2031 # The exports are deliberately scoped to this subshell. + export FMTG_TG_BIN="$tg" + telegram_away_deliver "$state" 'blocked: needs the ledger' 0 1 0 not-this-batch + ) + [ "$out" = 'unavailable|none' ] \ + || fail "a send naming another batch must refuse (got: $out)" + + rm -f "$state/.subsuper-escalations.since" + out=$( + # shellcheck disable=SC2030,SC2031 # The exports are deliberately scoped to this subshell. + export FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live + # shellcheck disable=SC2030,SC2031 # The exports are deliberately scoped to this subshell. + export FMTG_TG_BIN="$tg" + telegram_away_deliver "$state" 'blocked: needs the ledger' 0 1 0 1700000000-abcdef0123456789 + ) + [ "$out" = 'unavailable|none' ] \ + || fail "a send with no readable ledger must refuse (got: $out)" + + assert_absent "$home/tg-sent.log" "an untracked send must never reach the phone" + assert_absent "$state/tg-away-delivery" "an untracked send must not mint delivery evidence" + pass "away delivery refuses every send that does not come through the ledger" +} + +# A failure that provably never left the box must be genuinely retryable: the +# retired attempt keeps its evidence, the ordinal advances, and the very same lines +# reach the phone on the next flush once the client is back. +test_away_proven_local_failure_retries_the_same_lines() { + local home state tg injected first_evidence ledger + home=$(make_home away-proven-local-retry) + state="$home/state" + tg=$(make_fake_tg "$home") + : > "$state/.afk" + injected="$home/injected.log" + escalate_add "$state" 'blocked: needs the captain, client offline' + + # The client is not runnable yet, and the in-session receipt is wedged too, so + # the batch survives with its ledger. + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { return 1; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FM_TG_AWAY_RETRY_SECS=3600 FMTG_TG_BIN="$home/missing-tg" escalate_flush "$state" + ) && fail "a wedged receipt must leave escalate_flush unconfirmed" + assert_absent "$home/tg-sent.log" "a missing client cannot have sent anything" + ledger_field "$state" 2 0 "a proven-local failure must release its reservation" + ledger_field "$state" 5 1 "a proven-local failure must retire its attempt" + # shellcheck disable=SC2012 # Delivery evidence names are generated ids without spaces. + first_evidence=$(ls "$state/tg-away-delivery"/*.status | head -n 1) + grep -Eq '^unavailable [0-9]+ sender_missing$' "$first_evidence" \ + || fail "the retired attempt must keep its durable evidence" + + # The retry is scheduled, not immediate: while the backoff is unelapsed the flush + # neither contacts the phone nor mints another evidence record. + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" > "$injected"; return 1; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FM_TG_AWAY_RETRY_SECS=3600 FMTG_TG_BIN="$tg" escalate_flush "$state" + ) && fail "the still-wedged receipt must leave escalate_flush unconfirmed" + assert_absent "$home/tg-sent.log" "a deferred retry must not contact the phone" + assert_grep 'Telegram delivery deferred' "$injected" \ + "a deferred retry must say so in the in-session fallback" + # shellcheck disable=SC2012 # Delivery evidence names are generated ids without spaces. + [ "$(ls "$state/tg-away-delivery"/*.status | wc -l | tr -d ' ')" -eq 1 ] \ + || fail "a deferred retry must not mint another evidence record per tick" + ledger_field "$state" 5 1 "a deferred retry must not advance the attempt ordinal" + + # The client comes back and the schedule elapses. The same buffer, unchanged, + # must now reach the phone. + ledger=$(away_ledger_read "$state/.subsuper-escalations") + # shellcheck disable=SC2086 + set -- $ledger + away_ledger_write "$state/.subsuper-escalations" "$1" "$2" "$3" "$4" "$5" 0 "$7" \ + || fail "could not elapse the retry schedule" + ( + # shellcheck disable=SC2329 # Runtime override invoked by the code under test. + inject_msg() { printf '%s\n' "$1" > "$injected"; return 0; } + FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_TG_AWAY_EXEC=live \ + FMTG_TG_BIN="$tg" escalate_flush "$state" + ) || fail "the retry must complete the away flush" + assert_grep 'needs the captain, client offline' "$home/tg-sent.log" \ + "a released attempt must be able to reach the phone again" + [ "$(grep -c '^---$' "$home/tg-sent.log")" -eq 1 ] \ + || fail "the retry must contact the phone exactly once" + assert_grep 'Telegram accepted away-mode alert' "$injected" \ + "the successful retry must report its real accepted outcome" + [ -f "$first_evidence" ] \ + || fail "the retried attempt must not erase the earlier attempt's evidence" + pass "a proven-local failure retires its attempt and retries the same lines" +} + +# The counts are validated one by one before any comparison, sender call, or ledger +# mutation, so a partially-empty argument list cannot mint an id or send. +test_away_delivery_refuses_malformed_counts() { + local home state batch_id tg + home=$(make_home away-malformed-counts) + state="$home/state" + tg=$(make_fake_tg "$home") + : > "$state/.afk" + escalate_add "$state" 'blocked: must not slip through' + batch_id=$(away_ledger_read "$state/.subsuper-escalations" | cut -d' ' -f1) + + malformed_refused() { #