diff --git a/.pi/extensions/fm-primary-pi-watch.ts b/.pi/extensions/fm-primary-pi-watch.ts index 95a7eedd8d6..49f112a53ca 100644 --- a/.pi/extensions/fm-primary-pi-watch.ts +++ b/.pi/extensions/fm-primary-pi-watch.ts @@ -54,6 +54,17 @@ type SessionGeneration = { retryFailures: number; restoring: boolean; seq: number; + wakeBatchTimer: ReturnType | null; + wakeBatch: WakeBatchItem[]; + wakeBatchArm: ChildProcess | null; +}; + +type WakeRecovery = { generation: string; watcherPid: string }; + +type WakeBatchItem = { + key: string; + messages: string[]; + recoveries: WakeRecovery[]; }; function refreshWatchToolShell( @@ -96,6 +107,8 @@ const armReadyTimeoutMs = positiveInteger( process.platform === "win32" ? 35000 : 12000, ); const armRetireTimeoutMs = positiveInteger("FM_WATCH_ARM_RETIRE_TIMEOUT_MS", 1000); +const wakeBatchLimit = positiveInteger("FM_WAKE_BATCH_LIMIT", 20); +const wakeBatchSeconds = configuredWakeBatchSeconds(); const repairOnlyHint = "call fm_watch_arm_pi again only after a later notification says the cycle is missing, failed, or unhealthy"; const shuttingDownMessage = "watcher: not armed - Pi session is shutting down"; @@ -111,6 +124,18 @@ function positiveInteger(name: string, fallback: number): number { return Math.floor(value); } +function configuredWakeBatchSeconds(): number { + const override = Number(process.env.FM_WAKE_BATCH_SECONDS); + if (Number.isFinite(override) && override > 0) return Math.floor(override); + try { + const value = Number(readFileSync(`${config}/wake-batch-seconds`, "utf8").trim()); + if (Number.isFinite(value) && value > 0) return Math.floor(value); + } catch { + // An absent or unreadable local override keeps the safe shared default. + } + return 60; +} + function parentPid(pid: string): string { const result = spawnSync("ps", ["-o", "ppid=", "-p", pid], { encoding: "utf8" }); if (result.status !== 0) return ""; @@ -194,6 +219,9 @@ function createGeneration(): SessionGeneration { retryFailures: 0, restoring: false, seq: 0, + wakeBatchTimer: null, + wakeBatch: [], + wakeBatchArm: null, }; } @@ -209,6 +237,10 @@ function stopGeneration(generation: SessionGeneration): void { generation.stopping = true; if (generation.retryTimer) clearTimeout(generation.retryTimer); generation.retryTimer = null; + if (generation.wakeBatchTimer) clearTimeout(generation.wakeBatchTimer); + generation.wakeBatchTimer = null; + generation.wakeBatch = []; + generation.wakeBatchArm = null; if (generation.child) generation.child.kill("SIGTERM"); generation.child = null; } @@ -250,6 +282,139 @@ export default function (pi: ExtensionAPI) { await pi.sendUserMessage(content, { deliverAs: "followUp" }); } + function wakeIsUrgent(message: string): boolean { + const urgent = /(?:^|\n)(?:failed|blocked|needs-decision)(?:\s+\[[^\]]+\])?:|watcher: FAILED|(?:lost|no longer owns?) (?:the )?(?:session )?lock|lock ownership (?:was )?lost/i; + if (urgent.test(message)) return true; + const statusPaths = message.match(/[^\s()]+\.status\b/g) ?? []; + for (const statusPath of statusPaths) { + const path = resolve(statusPath); + if (!path.startsWith(`${resolve(state)}/`)) continue; + try { + const last = readFileSync(path, "utf8").trim().split(/\r?\n/).pop() ?? ""; + if (/^(?:failed|blocked|needs-decision)(?:\s+\[[^\]]+\])?:/.test(last)) return true; + } catch { + // Missing or unreadable endpoints are surfaced by the watcher itself. + } + } + return false; + } + + function wakeIdentity(message: string): string { + const leadingEndpoint = message.match(/^stale:\s*([^\s(]+)/)?.[1]; + const identities = [ + ...(message.match(/(?:^|[\s(])[^\s()]+\.(?:status|turn-ended)\b/g) ?? []), + ...(message.match(/\b(?:[A-Za-z0-9._-]+:)?w[A-Za-z0-9._-]+:p[A-Za-z0-9._-]+\b/g) ?? []), + ...(leadingEndpoint ? [leadingEndpoint] : []), + ].map((value) => value.trim()).sort(); + const wakeClass = message.match(/^(signal|stale|check|heartbeat)/)?.[1] ?? "wake"; + return identities.length > 0 ? `${wakeClass}:${[...new Set(identities)].join("|")}` : message.trim(); + } + + // is set only by the arm-end flush. bin/fm-watch-arm.sh prints + // its `watcher: started pid=...` line and then waits on that watcher, so an arm + // process only closes AFTER its own watcher pid has exited. Confirming a handling + // handshake there would always fail --handling-delivered's liveness gate and + // report a watcher failure that never happened, so an ended arm skips the + // confirmation: there is nothing left to confirm. A timer-driven flush still + // confirms, and a dangling arm that outlived its watcher is still retired below. + async function flushWakeBatch(owner: SessionGeneration, owningArmEnded = false): Promise { + if (!generationIsLive(owner) || owner.wakeBatch.length === 0) return; + if (owner.wakeBatchTimer) clearTimeout(owner.wakeBatchTimer); + owner.wakeBatchTimer = null; + const items = owner.wakeBatch.splice(0, owner.wakeBatch.length); + owner.wakeBatchArm = null; + const urgentDetails: string[] = []; + const routineDetails: string[] = []; + let recovery: WakeRecovery | undefined; + for (const item of items) { + for (const message of item.messages) { + const detail = message.length > 800 ? `${message.slice(0, 785)} [truncated]` : message; + (wakeIsUrgent(message) ? urgentDetails : routineDetails).push(detail); + } + if (item.recoveries.length > 0) recovery = item.recoveries[item.recoveries.length - 1]; + } + if (recovery && !owningArmEnded) { + const confirmed = confirmHandlingDeliveryWithRetry(owner, recovery); + if (!confirmed.ok) { + urgentDetails.push(confirmed.detail); + // A successor whose watcher died before confirming leaves owner.child + // pointing at an arm that will never supervise. Retiring it here keeps + // the dangling-arm recovery the pre-batching delivery path owned. + if (!pidAlive(recovery.watcherPid)) await retireArm(owner.child); + } + } + // An urgent wake is what triggered this flush, and it is appended last. Rendering + // in insertion order would push it past wakeBatchLimit behind the routine wakes it + // interrupted, so the follow-up would omit the very failure that broke the window. + // Urgent details are therefore rendered first and only routine details are omitted. + const details = [...urgentDetails, ...routineDetails]; + const omitted = Math.max(0, details.length - wakeBatchLimit); + const shown = details.slice(0, wakeBatchLimit); + const message = shown.length === 1 && omitted === 0 + ? shown[0] + : `batched ${details.length} watcher wakes:\n${shown.map((item) => `- ${item}`).join("\n")}${omitted > 0 ? `\n- ${omitted} more omitted` : ""}`; + await sendWake(owner, message); + } + + // A batch must not outlive the arm that opened it while that arm is NOT being + // replaced: an actionable close immediately starts a successor and carries the + // supervision chain forward, so the batch keeps aggregating across it - that + // rotation is the ordinary case the wake-batch window exists for. A + // non-actionable close hands off to a bounded retry instead, so the batch is + // delivered here rather than waiting out a window under a cycle that may never + // come back. The durable wake queue and the recovery marker remain the only + // records of what was delivered and acknowledged, so a crash between this flush + // and its confirmation re-presents on the next drain rather than losing it. + function flushWakeBatchOnArmEnd(owner: SessionGeneration, armChild: ChildProcess): void { + if (owner.wakeBatchArm !== armChild) return; + if (owner.wakeBatch.length === 0) { + owner.wakeBatchArm = null; + return; + } + void flushWakeBatch(owner, true).catch(() => { + // Pi owns delivery errors; durable wakes remain available to the drain. + }); + } + + async function queueWake(owner: SessionGeneration, message: string, recovery?: WakeRecovery): Promise { + if (!generationIsLive(owner)) return; + const key = wakeIdentity(message); + const existing = owner.wakeBatch.find((item) => item.key === key); + if (existing) { + // Identity collapse dedupes an unchanged repeat of the same wake, not a + // differently-typed one: two reasons for one endpoint (a paused recheck and + // then a vanished endpoint) must both reach the batch, or the second reason + // would only ever be seen by the drain. + if (!existing.messages.includes(message)) existing.messages.push(message); + if (recovery && !existing.recoveries.some((item) => item.generation === recovery.generation && item.watcherPid === recovery.watcherPid)) { + existing.recoveries.push(recovery); + } + } else { + owner.wakeBatch.push({ key, messages: [message], recoveries: recovery ? [recovery] : [] }); + } + // A batch is owned by the CURRENT arm, not the one that opened it. Every wake + // in an aggregating batch arrives just after an actionable rotation started a + // successor, so pinning ownership to the opening arm would leave it pointing at + // a predecessor that has already closed and can never end again - and the + // arm-end flush would then be unreachable for exactly the multi-wake batches + // batching exists for. A wake queued with no live arm leaves the last live + // owner in place rather than orphaning the batch. + if (owner.child) owner.wakeBatchArm = owner.child; + if (wakeIsUrgent(message)) { + await flushWakeBatch(owner); + return; + } + if (!owner.wakeBatchTimer) { + owner.wakeBatchTimer = setTimeout(() => { + owner.wakeBatchTimer = null; + void flushWakeBatch(owner).catch(() => { + // Pi owns delivery errors; durable wakes remain available to the drain. + }); + }, wakeBatchSeconds * 1000); + owner.wakeBatchTimer.unref(); + } + } + function confirmHandlingDelivery(recovery: { generation: string; watcherPid: string }): { ok: boolean; detail: string; @@ -295,25 +460,14 @@ export default function (pi: ExtensionAPI) { async function deliverActionableWake( owner: SessionGeneration, message: string, - recovery?: { generation: string; watcherPid: string }, + recovery?: WakeRecovery, ): Promise { if (!generationIsLive(owner)) return; - if (recovery) { - const confirmed = confirmHandlingDeliveryWithRetry(owner, recovery); - if (!confirmed.ok) { - const watcherPid = recovery.watcherPid; - if (!pidAlive(watcherPid)) { - await retireArm(owner.child); - } - await sendWake(owner, `${message}\n\n${confirmed.detail}`); - return; - } - } - await sendWake(owner, message); + await queueWake(owner, message, recovery); } function surfaceFailure(owner: SessionGeneration, message: string): void { - void sendWake(owner, message).catch(() => { + void queueWake(owner, message).catch(() => { // Pi owns delivery errors; continuity restoration never waits on prompting. }); } @@ -517,6 +671,7 @@ export default function (pi: ExtensionAPI) { return; } if (owner.restoring) return; + flushWakeBatchOnArmEnd(owner, armChild); scheduleRetry(owner, classification.message, predecessor); }); armChild.on("error", (error: Error) => { @@ -524,6 +679,7 @@ export default function (pi: ExtensionAPI) { settled = true; resolveClosed(); settleReadiness(false); + flushWakeBatchOnArmEnd(owner, armChild); releaseChild(); if (!generationIsLive(owner)) return; if (owner.restoring) return; diff --git a/.pi/extensions/fm-primary-turnend-guard.ts b/.pi/extensions/fm-primary-turnend-guard.ts index 1b2a3ec39ae..b8c5df81331 100644 --- a/.pi/extensions/fm-primary-turnend-guard.ts +++ b/.pi/extensions/fm-primary-turnend-guard.ts @@ -207,8 +207,8 @@ export default function (pi: ExtensionAPI) { await injectSessionstart(pi, source); }); - // Pi's compaction equivalent. The digest is what a compacted session has just - // lost, so re-emitting it here is the point rather than a side effect. + // Pi's compaction equivalent. The source owner routes this to the compact + // recovery digest: ownership, actionable queue, task identities, next step. pi.on?.("session_compact", async () => { await injectSessionstart(pi, "compact"); }); diff --git a/AGENTS.md b/AGENTS.md index 7c02933b462..7d92229d8ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,6 +77,7 @@ config/cmux-socket-password optional cmux control-socket password; LOCAL, gitig config/wedge-alarm optional away-mode wedge-alarm active-alert directives; LOCAL, gitignored; absent means auto (macOS Notification Center when available); see docs/wedge-alarm.md config/watched-tools.json optional list of the tools this home depends on, read by the update check armed with bin/fm-tool-update-check.sh; LOCAL, gitignored, firstmate-maintained but human-editable, and NOT inherited by secondmate homes; see docs/configuration.md "Watched tool updates" config/attended-routine-status-absorb optional "off" opt-out from default-on attended absorption of routine supervision wakes; LOCAL, gitignored; see docs/configuration.md "Attended routine status absorption" +config/wake-batch-seconds optional positive-integer Pi watcher follow-up aggregation window; LOCAL, gitignored, default 60 seconds; see docs/configuration.md "Pi watcher wake batching" config/x-mode.env generated Relay watcher cadence; LOCAL, gitignored; source before arming watcher when present data/ personal fleet records; LOCAL, gitignored as a whole backlog.md task queue, dependencies, history @@ -123,11 +124,12 @@ state/ runtime records and signals; gitignored ..open-decisions-cursor per-task byte cursor and folded open-decision set bounding the OPEN DECISIONS scan's cost to new status-log appends; written only by fm-classify-lib.sh's status_open_decisions_incremental, removed by teardown, safe to delete (forces one full re-fold) .status-presentation-cursor .status-presentation-lock fleet-wide per-task status identity/byte-offset manifest and serialization lock preventing already-presented status lines from being replayed as new; owned by fm-classify-lib.sh, with each task's row retired by teardown .status-absorbed- per-task receipt binding the status-file identity and byte endpoint the attended watcher absorbed instead of waking on; owned by fm-classify-lib.sh, retired once the drain presents those bytes, removed by teardown; never touch + .open-decisions-presentation fleet-wide receipt of the last OPEN DECISIONS block a reader was actually shown; lets an unchanged repeat collapse to a count marker while session-recovery and mechanical drains leave it unspent; owned by fm-wake-drain.sh, safe to delete (forces one full re-print) .afk durable away-mode flag; present = sub-supervisor may inject escalations (set by /afk, cleared on user return) .watch.lock .wake-queue.lock watcher singleton and queue serialization locks .claude-autoarm.lock .claude-autoarm-epoch .claude-autoarm-failure-notified .claude-autoarm-failure-alarmed .turnend-claude-blocks .turnend-claude-blocks.lock Claude Stop auto-arm single-flight, epoch, failure-episode, attended-alarm, guard-budget, and budget-lock records; never touch .cursor-park-owner .cursor-park-owner.lock .turnend-cursor-blocks Cursor stop-hook owner record, publication and commit lock, and bounded repair-nag budget; never touch - .hash-* .count-* .stale-* .stale-since-* .paused-* .wedge-escalations-* .writing-* .seen-* .hb-surfaced-* .last-* .heartbeat-streak watcher internals; never touch + .hash-* .count-* .stale-* .stale-since-* .paused-* .endpoint-missing-* .wedge-escalations-* .writing-* .seen-* .hb-surfaced-* .last-* .heartbeat-streak watcher internals; never touch .watch-triage.log watcher's absorbed-wake debug log (size-capped); never relied on, safe to delete .last-watcher-beat watcher liveness beacon, touched every poll (including while absorbing benign wakes); guard scripts read it .subsuper-* .supervise-daemon.* sub-supervisor internals; never touch @@ -165,6 +167,7 @@ When that section reports its checks still in progress it names exactly what is 3. **Wake queue** - when locked, presents the durable wake queue and prints the raw records prominently as this turn's first work queue; a clearly labeled status-event annotation may follow a valid `signal` record and includes every status line still unread at the presentation cursor, but never replaces the raw record or current-state reconciliation, and a lapsed watcher chain still surfaces here via the same guard alarm. Presented records remain durable until the handling turn runs the generation-bound acknowledgement printed by the drain. Every locked drain also prints a bounded fleet-wide `OPEN DECISIONS` section when durable decision records remain open, including when the queue itself is empty; reconcile those entries before continuing. + A later drain in the same session may print `OPEN DECISIONS: unchanged, N open` instead: that means the block you were already shown still stands unchanged, and it is just as actionable; a recovery drain (session start, `/clear`, compaction, away-mode return, or a drain that handles a watcher-down recovery episode) always prints the full block again. The same drain prints every still-unread `note:` line, pending-reply resolution, and routine status line the attended watcher absorbed instead of waking on, since the last presentation, in an unbounded `UNREAD STATUS` section, so an answer buried under a later routine line is not dropped; those lines are not re-printed after that presentation. It also prints a bounded `RECORD DIVERGENCE` section naming every captain call the status log reads as resolved while its backlog task is still held; nothing is closed for you, and `captain-hold-lifecycle` owns the reconciliation. When the lock could not be acquired and verified, the queue is left untouched because no session mutation is authorized, and the guard's tangle/watcher-liveness alarms still print in read-only advisory mode without drain, supervision repair, or checkout repair commands. diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index c5f270bdaf9..a0fdca023d6 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -2780,6 +2780,31 @@ fm_backend_herdr_queued_enter_busy() { # fi } +# Re-read an inconclusive post-Enter surface without sending another Enter. +# A later proven-pending composer authorizes the caller's bounded submit retry; +# a later empty composer or idle-baseline busy transition confirms delivery. +# Persistent unreadability remains unknown and never receives a blind keypress. +fm_backend_herdr_recheck_unknown_submit() { # + local target=$1 allow_busy=$2 sleep_s=$3 attempts i=0 raw verdict + attempts=${FM_BACKEND_HERDR_UNKNOWN_RECHECKS:-3} + case "$attempts" in ''|*[!0-9]*|0) attempts=3 ;; esac + while [ "$i" -lt "$attempts" ]; do + sleep "$sleep_s" + raw=$(fm_backend_herdr_agent_status_raw "$FM_BACKEND_HERDR_SESSION" "$FM_BACKEND_HERDR_PANE") + if [ "$allow_busy" = 1 ] \ + && [ "$(fm_backend_herdr_classify_submit_agent_status "$raw")" = busy ]; then + printf 'empty' + return 0 + fi + verdict=$(fm_backend_herdr_composer_state "$target") + case "$verdict" in + empty|pending|pending-unproven) printf '%s' "$verdict"; return 0 ;; + esac + i=$((i + 1)) + done + printf 'unknown' +} + fm_backend_herdr_send_text_submit() { # local target=$1 text=$2 retries=$3 sleep_s=$4 settle=$5 i=0 verdict baseline confirm_sleep local raw_status footer_baseline='' allow_rendered=0 enter_sent=0 @@ -2811,18 +2836,36 @@ fm_backend_herdr_send_text_submit() { # if [ "$baseline" = idle ]; then verdict=$(fm_backend_herdr_wait_for_working "$FM_BACKEND_HERDR_SESSION" "$FM_BACKEND_HERDR_PANE" \ "$confirm_sleep" "$FM_BACKEND_HERDR_SUBMIT_POLLS") - case "$verdict" in - busy) printf 'empty'; return 0 ;; - unknown) printf 'unknown'; return 0 ;; - esac - # Native stayed idle. Composer empty is positive delivery (a landed - # Claude turn that never flipped agent_status). Proven pending retries. - verdict=$(fm_backend_herdr_composer_state "$target") - case "$verdict" in - empty) printf 'empty'; return 0 ;; - pending|pending-unproven) ;; - *) printf '%s' "$verdict"; return 0 ;; - esac + if [ "$verdict" = unknown ]; then + # The recheck already read the composer boundedly; a pending verdict from + # it is the proof this retry needs, so it is used directly. Re-reading the + # composer here would let a single transient unknown from that second read + # print an unbounded 'unknown' the recheck exists to prevent. + verdict=$(fm_backend_herdr_recheck_unknown_submit "$target" 1 "$sleep_s") + case "$verdict" in + empty) printf 'empty'; return 0 ;; + unknown) printf 'unknown'; return 0 ;; + esac + else + case "$verdict" in + busy) printf 'empty'; return 0 ;; + esac + # Native stayed idle. Composer empty is positive delivery (a landed + # Claude turn that never flipped agent_status). Proven pending retries. + verdict=$(fm_backend_herdr_composer_state "$target") + case "$verdict" in + empty) printf 'empty'; return 0 ;; + pending|pending-unproven) ;; + unknown) + verdict=$(fm_backend_herdr_recheck_unknown_submit "$target" 1 "$sleep_s") + case "$verdict" in + empty) printf 'empty'; return 0 ;; + unknown) printf 'unknown'; return 0 ;; + esac + ;; + *) printf '%s' "$verdict"; return 0 ;; + esac + fi else sleep "$sleep_s" verdict=$(fm_backend_herdr_composer_state "$target") @@ -2834,7 +2877,10 @@ fm_backend_herdr_send_text_submit() { # case "$verdict" in busy) printf 'empty'; return 0 ;; empty) printf 'empty'; return 0 ;; - unknown) printf 'unknown'; return 0 ;; + unknown) + verdict=$(fm_backend_herdr_recheck_unknown_submit "$target" 0 "$sleep_s") + case "$verdict" in empty) printf 'empty'; return 0 ;; unknown) printf 'unknown'; return 0 ;; esac + ;; esac fi i=$((i + 1)) diff --git a/bin/fm-afk-return.sh b/bin/fm-afk-return.sh index 88fb27792c4..5249a1561cf 100755 --- a/bin/fm-afk-return.sh +++ b/bin/fm-afk-return.sh @@ -153,7 +153,7 @@ return_reconcile() { fi fi - drained=$("$SCRIPT_DIR/fm-wake-drain.sh" 2> "$drain_err") || { + drained=$("$SCRIPT_DIR/fm-wake-drain.sh" --session-recovery 2> "$drain_err") || { append_evidence lifecycle 'durable wake drain failed; retry catch-up before ordinary work' "$evidence" lifecycle_ok=0 drained="" diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 2bc8c944961..4d4aa9c813c 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -84,6 +84,10 @@ FM_CLASSIFY_PAUSED_VERB_DEFAULT='paused' # one owner. # shellcheck disable=SC2034 # Read by the watcher and daemon (fm-watch.sh, fm-supervise-daemon.sh), not this lib. FM_PAUSE_RESURFACE_SECS_DEFAULT=3600 +# shellcheck disable=SC2034 # Read by fm-watch.sh after sourcing this policy owner. +FM_PAUSED_RESURFACE_BATCH_LIMIT_DEFAULT=20 +# shellcheck disable=SC2034 # Read by fm-watch.sh after sourcing this policy owner. +FM_ENDPOINT_BATCH_LIMIT_DEFAULT=20 # The resolution verb and durable-backlog-transfer verb that CLOSE a keyed # status decision opened by needs-decision or blocked. See status_open_decisions @@ -207,6 +211,18 @@ status_is_paused_or_captain_held() { # # The parsers are pure reads of a single line. Status metadata may contain any # number of "[name=value]" tags before the colon, in any order, so verb parsing # ends at the first tag rather than special-casing "[key=...]". +# The status verbs that make a wake URGENT rather than routine: a supervisor must +# see them without waiting out any aggregation window. Kept beside status_line_verb +# as the one owner of that set, so the watcher's stale reasons and the Pi +# extension's urgent bypass cannot disagree about which verbs qualify. A declared +# wait (paused:) and ordinary progress (working:) are deliberately not urgent. +status_line_is_urgent() { # + case "$(status_line_verb "$1")" in + failed|blocked|needs-decision) return 0 ;; + esac + return 1 +} + status_line_verb() { # -> leading verb word local v=${1%%:*} v=${v%%\[*} @@ -1531,8 +1547,27 @@ crew_worktree_written_since() { # # raised decision, a mirrored remote line), and a busy mate agent makes its note # more current, not less deliverable. Scoped to .status files - a mate's bare # turn-ended ping still uses the ordinary provably-working absorb. +# The ONE owner of the mate routed-reply carve-out described directly above, so +# every absorber that consults it agrees. 0 when any listed file is a +# kind=secondmate task's .status; a mate's bare turn-ended ping is not one. +signal_list_has_secondmate_status() { # ... + local f base dir task + for f in "$@"; do + base=${f##*/} + dir=${f%/*} + [ "$dir" != "$f" ] || dir=. + case "$base" in *.status) task=${base%.status} ;; *) continue ;; esac + [ -n "$task" ] || continue + if [ "$(grep '^kind=' "$dir/$task.meta" 2>/dev/null | tail -1 | cut -d= -f2-)" = secondmate ]; then + return 0 + fi + done + return 1 +} + signal_crew_provably_working() { # ... local f base dir task seen="" + signal_list_has_secondmate_status "$@" && return 1 for f in "$@"; do base=${f##*/} dir=${f%/*} @@ -1543,13 +1578,6 @@ signal_crew_provably_working() { # ... *) continue ;; esac [ -n "$task" ] || continue - case "$base" in - *.status) - if [ "$(grep '^kind=' "$dir/$task.meta" 2>/dev/null | tail -1 | cut -d= -f2-)" = secondmate ]; then - return 1 - fi - ;; - esac case " $seen " in *" $task "*) continue ;; esac seen="$seen $task" crew_is_provably_working "$task" || return 1 diff --git a/bin/fm-primary-scope-lib.sh b/bin/fm-primary-scope-lib.sh index 536e62e7ab7..32dad950ff9 100755 --- a/bin/fm-primary-scope-lib.sh +++ b/bin/fm-primary-scope-lib.sh @@ -17,6 +17,35 @@ fm_root_is_secondmate_home() { return 0 } +# Return 0 only when is a linked worktree registered as a live crew task +# in the primary checkout that owns its git common directory. +fm_root_is_registered_crew_worktree() { # + local root=$1 git_dir git_common primary meta recorded resolved_root + fm_root_is_secondmate_home "$root" && return 1 + git_dir=$(git -C "$root" rev-parse --git-dir 2>/dev/null) || return 1 + git_common=$(git -C "$root" rev-parse --git-common-dir 2>/dev/null) || return 1 + [ "$git_dir" != "$git_common" ] || return 1 + case "$git_common" in + /*) ;; + *) git_common=$(cd "$root" && cd "$git_common" 2>/dev/null && pwd -P) || return 1 ;; + esac + case "$git_common" in */.git) primary=${git_common%/.git} ;; *) return 1 ;; esac + resolved_root=$(cd "$root" 2>/dev/null && pwd -P) || return 1 + for meta in "$primary"/state/*.meta; do + [ -f "$meta" ] && [ ! -L "$meta" ] || continue + recorded=$(awk -F= '$1 == "worktree" { sub(/^[^=]*=/, ""); print; exit }' "$meta" 2>/dev/null) || continue + [ -n "$recorded" ] || continue + [ -d "$recorded" ] || continue + recorded=$(cd "$recorded" 2>/dev/null && pwd -P) || continue + [ "$recorded" = "$resolved_root" ] && return 0 + done + return 1 +} + +fm_print_crew_worktree_suppression() { + printf '%s\n' 'crew worktree - digest suppressed' +} + # Return 0 when $1 is a genuine primary root whose effective state dir is $2. # A valid secondmate marker force-includes a linked secondmate home. # Otherwise only a plain checkout is primary, never a linked task worktree. diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index ba9d5ccef3d..d118db5c921 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -178,14 +178,14 @@ # Hosts without timeout, gtimeout, or perl use the shared pure-Bash watchdog, so # the digest never runs without the same hard bound and process-group cleanup. # -# Usage: fm-session-start.sh [--reemit] [--source ] +# Usage: fm-session-start.sh [--reemit|--compact] [--source ] # Prints the full ordered digest to stdout and always exits 0: this is a # reporting command, not a gate. A lock refusal is reported as a loud # banner inline, never a silent failure or a non-zero exit that would make # an agent skip the rest of the digest. # # --reemit This process ALREADY took the helm at its own startup and has -# only lost its context (a /clear or a compaction). Skip the +# only lost its context after a /clear. Skip the # mutating sweeps that startup already reconciled - the stale Herdr # projection cleanup and bootstrap's six mutating sweeps (fleet # sync, secondmate convergence and liveness, PR-check migration, @@ -199,6 +199,12 @@ # proceeds, while a lock another live session took meanwhile still # produces the ordinary read-only path. # +# --compact Re-verify lock and watcher ownership, drain only the actionable +# queue and open decisions, print active task identities, then the +# exact next supervision instruction. Status tails, unread routine +# status, and unchanged context files are omitted; unread routine +# bytes remain unacknowledged for the next ordinary drain. +# # --source The native session-open source, supplied only by # fm-sessionstart-run.sh. A genuine `startup` that owns the active # session lock records AGENTS.md's SHA-256 baseline only after the @@ -222,6 +228,7 @@ COMPLETION_FILE="$STATE/.session-start-complete" AGENTS_BASELINE_FILE="$STATE/.session-start-agents-baseline" REEMIT=0 +COMPACT=0 SESSION_SOURCE= while [ "$#" -gt 0 ]; do case "$1" in @@ -229,6 +236,10 @@ while [ "$#" -gt 0 ]; do REEMIT=1 shift ;; + --compact) + COMPACT=1 + shift + ;; --source) SESSION_SOURCE=${2:-} if [ "$#" -ge 2 ]; then shift 2; else shift; fi @@ -243,7 +254,7 @@ while [ "$#" -gt 0 ]; do ;; *) printf 'fm-session-start: unknown argument: %s\n' "$1" >&2 - printf 'usage: fm-session-start.sh [--reemit] [--source ]\n' >&2 + printf 'usage: fm-session-start.sh [--reemit|--compact] [--source ]\n' >&2 exit 2 ;; esac @@ -271,13 +282,43 @@ if [ -z "${FM_SESSION_START_STAGE_FILE:-}" ]; then # the deadline outright), so an unusable value falls back to the default # rather than silently removing the bound. case "$SESSION_START_BUDGET" in ''|*[!0-9]*|0) SESSION_START_BUDGET=120 ;; esac + + # A worker in a registered worktree of this repository is not a fleet primary, + # so it emits the suppression line instead of a digest. The predicate shells + # out to git, which a stuck index.lock or an unresponsive filesystem can hang + # forever, so it runs bounded and only here in the parent - never inside the + # timed child, where it would spend the digest's whole budget before the first + # stage. Hitting the bound (or any failure) falls through to the digest: a + # loud, possibly redundant startup beats a silent one. + CREW_CHECK_BUDGET=5 + [ "$SESSION_START_BUDGET" -ge "$CREW_CHECK_BUDGET" ] || CREW_CHECK_BUDGET=$SESSION_START_BUDGET + # shellcheck source=bin/fm-primary-scope-lib.sh + . "$SCRIPT_DIR/fm-primary-scope-lib.sh" + # shellcheck disable=SC2016 # $1/$2 are the child bash -c positional args, expanded there, not here. + if fm_run_timed "$CREW_CHECK_BUDGET" bash -c \ + '. "$1/fm-primary-scope-lib.sh"; fm_root_is_registered_crew_worktree "$2"' \ + _ "$SCRIPT_DIR" "$FM_ROOT"; then + fm_print_crew_worktree_suppression + exit 0 + fi + SESSION_START_STAGE_FILE=$(mktemp "${TMPDIR:-/tmp}/fm-session-start-stage.XXXXXX" 2>/dev/null) || SESSION_START_STAGE_FILE= if [ -z "$SESSION_START_STAGE_FILE" ]; then # Without a breadcrumb the bound still holds; only the banner's precision # is lost, so the child still runs bounded. SESSION_START_STAGE_FILE=/dev/null fi - if [ "$REEMIT" -eq 1 ]; then + if [ "$COMPACT" -eq 1 ]; then + if [ -n "$SESSION_SOURCE" ]; then + fm_run_timed "$SESSION_START_BUDGET" \ + env FM_SESSION_START_STAGE_FILE="$SESSION_START_STAGE_FILE" \ + "$SCRIPT_DIR/fm-session-start.sh" --compact --source "$SESSION_SOURCE" + else + fm_run_timed "$SESSION_START_BUDGET" \ + env FM_SESSION_START_STAGE_FILE="$SESSION_START_STAGE_FILE" \ + "$SCRIPT_DIR/fm-session-start.sh" --compact + fi + elif [ "$REEMIT" -eq 1 ]; then if [ -n "$SESSION_SOURCE" ]; then fm_run_timed "$SESSION_START_BUDGET" \ env FM_SESSION_START_STAGE_FILE="$SESSION_START_STAGE_FILE" \ @@ -600,6 +641,62 @@ EOF fi } +if [ "$COMPACT" -eq 1 ]; then + printf 'COMPACT RECOVERY - %s\n' "$FM_HOME" + printf 'LOCK AND WATCHER OWNERSHIP\n' + COMPACT_LOCK_OUT=$("$SCRIPT_DIR/fm-lock.sh" 2>&1) + COMPACT_LOCK_RC=$? + printf 'lock: %s\n' "$COMPACT_LOCK_OUT" + REBUILDING_SESSION_PID=$(fm_harness_ancestry_pid 2>/dev/null || true) + print_agents_refresh_if_required "$REBUILDING_SESSION_PID" + WATCH_PID=$(cat "$STATE/.watch.lock/pid" 2>/dev/null || true) + case "$WATCH_PID" in + ''|*[!0-9]*) printf 'watcher: no readable owner pid\n' ;; + *) + if kill -0 "$WATCH_PID" 2>/dev/null; then + printf 'watcher: owned by live pid %s\n' "$WATCH_PID" + else + printf 'watcher: recorded owner pid %s is not live\n' "$WATCH_PID" + fi + ;; + esac + COMPACT_AFK_PRESENT=0 + [ -e "$STATE/.afk" ] && COMPACT_AFK_PRESENT=1 + COMPACT_X_MODE_PRESENT=0 + [ -f "$CONFIG/x-mode.env" ] && COMPACT_X_MODE_PRESENT=1 + "$SCRIPT_DIR/fm-supervision-instructions.sh" --harness "$PRIMARY_HARNESS" \ + --afk "$COMPACT_AFK_PRESENT" --x-mode "$COMPACT_X_MODE_PRESENT" --state-lines + printf 'ACTIONABLE QUEUE AND OPEN DECISIONS\n' + if [ "$COMPACT_LOCK_RC" -eq 0 ]; then + "$SCRIPT_DIR/fm-wake-drain.sh" --compact 2>&1 || true + else + printf 'queue drain skipped because this session does not own the lock\n' + fi + printf 'ACTIVE TASK IDENTITIES\n' + COMPACT_META_FOUND=0 + for meta in "$STATE"/*.meta; do + [ -f "$meta" ] && [ ! -L "$meta" ] || continue + COMPACT_META_FOUND=1 + id=$(basename "$meta" .meta) + kind=$(fm_meta_get "$meta" kind 2>/dev/null || true) + harness=$(fm_meta_get "$meta" harness 2>/dev/null || true) + backend=$(fm_backend_of_meta "$meta" 2>/dev/null || true) + target=$(fm_backend_target_of_meta "$meta" 2>/dev/null || true) + printf '%s kind=%s harness=%s backend=%s target=%s\n' \ + "$id" "${kind:-ship}" "${harness:-unknown}" "${backend:-unknown}" "${target:-unknown}" + done + [ "$COMPACT_META_FOUND" -eq 1 ] || printf '(none)\n' + printf 'NEXT SUPERVISION INSTRUCTION\n' + if [ "$COMPACT_LOCK_RC" -eq 0 ]; then + "$SCRIPT_DIR/fm-supervision-instructions.sh" --harness "$PRIMARY_HARNESS" \ + --afk "$COMPACT_AFK_PRESENT" --next-line + else + "$SCRIPT_DIR/fm-supervision-instructions.sh" --harness "$PRIMARY_HARNESS" \ + --read-only 1 --repair-line + fi + exit 0 +fi + AGENTS_START_HASH= if [ "$REEMIT" -eq 0 ] && [ "$SESSION_SOURCE" = startup ]; then AGENTS_START_HASH=$(hash_file_sha256 "$FM_ROOT/AGENTS.md" 2>/dev/null || true) @@ -692,7 +789,10 @@ fi # wake, without adding a daemon or external-network call. # Presented records are this turn's first work queue and remain durable until # post-handling acknowledgement. The drain's separate OPEN DECISIONS section -# remains actionable even when that queue is empty (AGENTS.md sections 3 and 8). +# remains actionable even when that queue is empty (AGENTS.md sections 3 and 8); +# --session-recovery is what keeps that true here, because a digest runs only when +# this session's context was lost (start, /clear re-emit, compact) and the drain's +# unchanged-decision collapse would otherwise carry across that boundary. # The drain also runs fm-guard.sh internally on the locked path, so the # tangle/watcher-liveness alarms land right here too, ahead of the bulk digest # below. The read-only path never touches the queue because it lacks mutation @@ -713,7 +813,7 @@ else if [ -n "$INACTIVE_OUT" ]; then printf 'inactive outcome reconciliation: %s\n' "$INACTIVE_OUT" fi - DRAIN_OUT=$("$SCRIPT_DIR/fm-wake-drain.sh" 2>&1) + DRAIN_OUT=$("$SCRIPT_DIR/fm-wake-drain.sh" --session-recovery 2>&1) if [ -n "$DRAIN_OUT" ]; then printf '%s\n' "$DRAIN_OUT" else diff --git a/bin/fm-sessionstart-nudge.sh b/bin/fm-sessionstart-nudge.sh index fccf775dd95..129dc1554e9 100755 --- a/bin/fm-sessionstart-nudge.sh +++ b/bin/fm-sessionstart-nudge.sh @@ -18,6 +18,10 @@ STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" . "$SCRIPT_DIR/fm-operational-input.sh" fm_is_gate_agent "$FM_ROOT" && exit 0 +if fm_root_is_registered_crew_worktree "$FM_ROOT"; then + fm_print_crew_worktree_suppression + exit 0 +fi fm_primary_scope_matches "$FM_ROOT" "$STATE" || exit 0 lock_is_in_ancestry() { diff --git a/bin/fm-sessionstart-run.sh b/bin/fm-sessionstart-run.sh index 50496eef295..825960fbf4c 100755 --- a/bin/fm-sessionstart-run.sh +++ b/bin/fm-sessionstart-run.sh @@ -19,9 +19,10 @@ # # Source routing (see docs/sessionstart-nudge.md for the per-harness names): # startup, new full digest - this process has not taken the helm -# clear, compact `--reemit` digest only when this lock owner recorded -# a completed full startup; otherwise a full digest, -# so a startup killed mid-sweep is finished first +# compact compact recovery digest only when this lock owner +# recorded a completed full startup; otherwise a full +# digest so a startup killed mid-sweep is finished first +# clear `--reemit` digest under the same completion gate # resume, reload, fork delegate to the nudge wrapper. Prior context is # restored on these, so re-running is redundant when # this process still holds the lock (the nudge stays @@ -69,6 +70,10 @@ done # agent and an unmarked task worktree can never run a session start for a home # they do not own. fm_is_gate_agent "$FM_ROOT" && exit 0 +if fm_root_is_registered_crew_worktree "$FM_ROOT"; then + fm_print_crew_worktree_suppression + exit 0 +fi fm_primary_scope_matches "$FM_ROOT" "$STATE" || exit 0 session_start_completed() { @@ -113,7 +118,14 @@ case "$SOURCE" in resume|reload|fork) exec "$SCRIPT_DIR/fm-sessionstart-nudge.sh" ;; - clear|compact) + compact) + if session_start_completed; then + "$SCRIPT_DIR/fm-session-start.sh" --compact --source "$SOURCE" || true + else + "$SCRIPT_DIR/fm-session-start.sh" --source "$SOURCE" || true + fi + ;; + clear) if session_start_completed; then "$SCRIPT_DIR/fm-session-start.sh" --reemit --source "$SOURCE" || true else diff --git a/bin/fm-supervise-daemon.sh b/bin/fm-supervise-daemon.sh index b61b04af537..19a0e11da29 100755 --- a/bin/fm-supervise-daemon.sh +++ b/bin/fm-supervise-daemon.sh @@ -1317,7 +1317,7 @@ handle_durable_wakes() { # local handled=0 ack_through ack_generation out=$(mktemp "$state/.subsuper-wake-drain.XXXXXX") || return 1 err=$(mktemp "$state/.subsuper-wake-drain.XXXXXX") || { rm -f "$out"; return 1; } - if ! "$FM_DAEMON_DIR/fm-wake-drain.sh" > "$out" 2> "$err"; then + if ! "$FM_DAEMON_DIR/fm-wake-drain.sh" --no-presentation-commit > "$out" 2> "$err"; then cat "$err" >&2 rm -f "$out" "$err" return 1 diff --git a/bin/fm-supervision-instructions.sh b/bin/fm-supervision-instructions.sh index a503bd9d35e..281a501b171 100755 --- a/bin/fm-supervision-instructions.sh +++ b/bin/fm-supervision-instructions.sh @@ -15,14 +15,19 @@ READ_ONLY=0 AFK=0 X_MODE=0 REPAIR_LINE=0 +NEXT_LINE=0 +STATE_LINES=0 QUEUE_PENDING=0 usage() { cat <<'EOF' -Usage: fm-supervision-instructions.sh [--harness ] [--read-only 0|1] [--afk 0|1] [--x-mode 0|1] [--repair-line] [--queue-pending 0|1] +Usage: fm-supervision-instructions.sh [--harness ] [--read-only 0|1] [--afk 0|1] [--x-mode 0|1] [--repair-line|--next-line|--state-lines] [--queue-pending 0|1] Print the current primary harness's supervision operating instructions. With --repair-line, print one concise repair instruction for guard and hook messages. +With --next-line, print the exact ordinary continuation after compact recovery. +With --state-lines, print only the away-mode and X-mode state lines, for a bounded +digest that must report who owns supervision without any bulk output. EOF } @@ -64,6 +69,14 @@ while [ "$#" -gt 0 ]; do REPAIR_LINE=1 shift ;; + --next-line) + NEXT_LINE=1 + shift + ;; + --state-lines) + STATE_LINES=1 + shift + ;; -h|--help) usage exit 0 @@ -158,28 +171,66 @@ repair_line() { esac } +# Away mode and X mode both change WHO owns supervision and what a wake means, so +# every digest that reports supervision state prints them from here rather than +# re-wording them. Two lines, no bulk output, so a bounded digest can carry them. +supervision_state_lines() { + if [ "$AFK" -eq 1 ]; then + printf '%s\n' '- Away mode: active; load /afk and keep normal harness supervision paused while the daemon owns the watcher.' + else + printf '%s\n' '- Away mode: inactive.' + fi + if [ "$X_MODE" -eq 1 ]; then + printf '%s%s%s\n' '- X mode: active; source ' "$x_mode_env" ' before launching any watcher process so the 30s cadence is inherited.' + else + printf '%s\n' '- X mode: inactive; use the default watcher cadence.' + fi +} + +# The repo-relative path of the protocol snippet this harness actually renders, +# derived from $SNIPPET so pi-signed resolves to pi.md and any unresolved harness +# resolves to unknown.md without a second mapping to keep in step. +protocol_doc_path() { + printf 'docs/supervision-protocols/%s' "${SNIPPET##*/}" +} + +# Every line here is SELF-CONTAINED: the drain-and-acknowledge step, the one-line +# condition, the exact command (or the reason no command is owed), and the path of +# the owning protocol document. --next-line prints this into a compact-recovery +# digest that carries no protocol snippet, so a session that just lost its context +# cannot follow a bare "as directed below" - and inlining the protocol itself would +# defeat the point of compaction. ordinary_wake_line() { + # Away mode changes WHO owns the queue, not just how a wake is delivered: + # bin/fm-supervise-daemon.sh drains and triages the same durable queue, so the + # attended drain-and-acknowledge order below would consume its work. One + # self-contained line, same shape as the harness lines: the exact action, the + # condition, what not to do, and the owning document. + if [ "$AFK" -eq 1 ]; then + printf '%s\n' '- Ordinary wake: away mode is active (state/.afk present) and bin/fm-supervise-daemon.sh owns supervision, so load the /afk skill and let the daemon triage this wake; do NOT run bin/fm-wake-drain.sh or its --ack-through command from here. Protocol: docs/architecture.md' + return 0 + fi case "$HARNESS" in claude) - printf '%s\n' '- Ordinary wake: the Stop-owned auto-arm (bin/fm-claude-stop-autoarm.sh) already owns watcher continuity; drain and handle the wake, and do not arm another cycle yourself.' + printf '%s%s\n' '- Ordinary wake: drain and handle this wake with bin/fm-wake-drain.sh, then run the exact --ack-through command it printed; the Stop-owned auto-arm (bin/fm-claude-stop-autoarm.sh) already owns watcher continuity, so do not arm another cycle yourself. Protocol: ' "$(protocol_doc_path)" ;; codex) - printf '%s\n' '- Ordinary wake: take the next foreground bin/fm-watch-checkpoint.sh checkpoint as directed below.' + printf '%s%s%s%s\n' '- Ordinary wake: drain and handle this wake with bin/fm-wake-drain.sh, then run the exact --ack-through command it printed; you own continuity here, so start the next foreground checkpoint with bin/fm-watch-checkpoint.sh --seconds ' "$checkpoint_seconds" ' and never use shell &. Protocol: ' "$(protocol_doc_path)" ;; pi|pi-signed) - printf '%s\n' '- Ordinary wake: the Pi extension already owns watcher continuity; do not arm another cycle.' + printf '%s%s\n' '- Ordinary wake: drain and handle this wake with bin/fm-wake-drain.sh, then run the exact --ack-through command it printed; the Pi extension already owns watcher continuity, so do not arm another cycle. Protocol: ' "$(protocol_doc_path)" ;; opencode) - printf '%s\n' '- Ordinary wake: the OpenCode TUI plugin already owns watcher continuity; do not arm manually.' + printf '%s%s\n' '- Ordinary wake: drain and handle this wake with bin/fm-wake-drain.sh, then run the exact --ack-through command it printed; the OpenCode TUI plugin already owns watcher continuity, so do not arm manually. Protocol: ' "$(protocol_doc_path)" ;; grok) - printf '%s\n' '- Ordinary wake: re-arm exactly one bin/fm-watch-arm.sh Grok tracked background task as directed below.' + printf '%s%s%s%s%s%s\n' '- Ordinary wake: drain and handle this wake with bin/fm-wake-drain.sh, then run the exact --ack-through command it printed; you own continuity here, so re-arm exactly one Grok tracked background task by calling run_terminal_command with background: true on `[ -f ' "$x_mode_env_sh" ' ] && . ' "$x_mode_env_sh" '; exec bin/fm-watch-arm.sh`, and never use shell &. Protocol: ' "$(protocol_doc_path)" ;; cursor) - printf '%s\n' '- Ordinary wake: the stop-hook park (bin/fm-turnend-guard-cursor.sh) already owns watcher continuity; drain and handle the wake, and do not arm another cycle yourself.' + printf '%s%s\n' '- Ordinary wake: drain and handle this wake with bin/fm-wake-drain.sh, then run the exact --ack-through command it printed; the stop-hook park (bin/fm-turnend-guard-cursor.sh) already owns watcher continuity, so do not arm another cycle yourself. Protocol: ' "$(protocol_doc_path)" ;; *) - printf '%s\n' '- Ordinary wake: follow the continuation in the harness protocol below; do not use shell &.' + printf '%s%s%s\n' '- Ordinary wake: drain and handle this wake with bin/fm-wake-drain.sh, then run the exact --ack-through command it printed; this harness has no verified wake adapter, so repeat the same bounded supervision wait it can actually wake from and never use shell &. Protocol: ' "$(protocol_doc_path)" ' and AGENTS.md' ;; esac } @@ -189,6 +240,16 @@ if [ "$REPAIR_LINE" -eq 1 ]; then exit 0 fi +if [ "$STATE_LINES" -eq 1 ]; then + supervision_state_lines + exit 0 +fi + +if [ "$NEXT_LINE" -eq 1 ]; then + ordinary_wake_line | sed 's/^- Ordinary wake: //' + exit 0 +fi + RULE='================================================================================' printf '%s\n' "$RULE" printf 'SUPERVISION OPERATING INSTRUCTIONS - primary harness: %s\n' "$HARNESS" @@ -199,16 +260,7 @@ if [ "$READ_ONLY" -eq 1 ]; then else printf '%s\n' '- Lock: held by this session; this session owns normal supervision unless away mode says otherwise.' fi -if [ "$AFK" -eq 1 ]; then - printf '%s\n' '- Away mode: active; load /afk and keep normal harness supervision paused while the daemon owns the watcher.' -else - printf '%s\n' '- Away mode: inactive.' -fi -if [ "$X_MODE" -eq 1 ]; then - printf '%s%s%s\n' '- X mode: active; source ' "$x_mode_env" ' before launching any watcher process so the 30s cadence is inherited.' -else - printf '%s\n' '- X mode: inactive; use the default watcher cadence.' -fi +supervision_state_lines ordinary_wake_line printf '\n' render_snippet diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 2da77a30a38..38ac4949d1f 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -29,9 +29,39 @@ ACK_THROUGH= ACK_GENERATION= ACK_FINGERPRINTS= ACK_NOTICE_FINGERPRINTS= +COMPACT=0 +# A session-recovery drain is one that runs precisely BECAUSE this session's +# context was lost - session start, /clear re-emit, and compaction. The +# unchanged-open-decisions collapse below assumes the full block is already in +# this session's context, which is exactly what those three events destroy, so a +# recovery drain re-presents it in full. An ordinary mid-turn wake drain runs in +# the same context that saw the last presentation and keeps the collapse. +SESSION_RECOVERY=0 +# Every presentation record this drain writes - the unchanged-open-decisions +# collapse AND the UNREAD STATUS cursor - means a HUMAN session was shown those +# bytes. A caller that consumes this drain's rows mechanically and discards the +# presented text has shown them to nobody, so it must spend neither: doing so +# collapses the decisions and swallows the unread span for whoever reads it next. +# The away-mode daemon is exactly that caller, and the captain returning from away +# mode is the reader those records were about to be spent on. +PRESENTATION_COMMIT=1 +OPEN_DECISIONS_PRESENTATION_PENDING= case "${1:-}" in '') ;; + --compact) + COMPACT=1 + SESSION_RECOVERY=1 + [ "$#" -eq 1 ] || { echo "wake drain: unexpected compact arguments" >&2; exit 2; } + ;; + --session-recovery) + SESSION_RECOVERY=1 + [ "$#" -eq 1 ] || { echo "wake drain: unexpected session-recovery arguments" >&2; exit 2; } + ;; + --no-presentation-commit) + PRESENTATION_COMMIT=0 + [ "$#" -eq 1 ] || { echo "wake drain: unexpected no-presentation-commit arguments" >&2; exit 2; } + ;; --ack-through) ACK_THROUGH=${2:-} case "$ACK_THROUGH" in ''|*[!0-9]*) echo "wake drain: invalid acknowledgement sequence" >&2; exit 2 ;; esac @@ -41,9 +71,23 @@ case "${1:-}" in case "$ACK_GENERATION" in ''|*[!A-Za-z0-9._-]*) echo "wake drain: invalid recovery generation" >&2; exit 2 ;; esac [ "$#" -eq 4 ] || { echo "wake drain: unexpected acknowledgement arguments" >&2; exit 2; } ;; - *) echo "usage: fm-wake-drain.sh [--ack-through SEQUENCE --recovery-generation GENERATION]" >&2; exit 2 ;; + *) echo "usage: fm-wake-drain.sh [--compact | --session-recovery | --no-presentation-commit | --ack-through SEQUENCE --recovery-generation GENERATION]" >&2; exit 2 ;; esac +OPEN_DECISIONS_FORCE= +[ "$SESSION_RECOVERY" -eq 0 ] || OPEN_DECISIONS_FORCE=force + +# A watcher-down recovery drain is the second context in which the collapse's +# premise fails. The recovery wake exists precisely because an unsupervised +# interval passed, so the decisions this drain re-folds were last presented to a +# session that is no longer supervising them - and a decision-only recovery +# (empty queue, still-open decision) would otherwise re-surface as a bare count +# with no key, task or resolve instruction, which is the entire payload of that +# wake. Set by the recovery paths below once the episode is known. +force_open_decisions_for_recovery() { + OPEN_DECISIONS_FORCE=force +} + # Defense in depth for the supervision chain: this script runs at the top of # every wake-handling and recovery turn, so assert supervision health here too. A # lapsed supervision chain then surfaces on a plain drain-and-handle turn, not @@ -115,6 +159,10 @@ EOF [ "$shown" -gt 0 ] || return 0 } +open_decisions_digest() { + git hash-object --stdin 2>/dev/null +} + # Print the consolidated OPEN DECISIONS section: every still-open # needs-decision/blocked, fleet-wide, folded from the durable status logs by # fm-classify-lib.sh's status_open_decisions fold (via its cursor-backed @@ -130,16 +178,35 @@ EOF # fm-classify-lib.sh's "incremental (cursor-backed) open-decisions fold"). # Bounded and silent: prints nothing when no decision is open, which is the # common case. +# , set by every session-recovery caller (see SESSION_RECOVERY above), +# skips the unchanged short-circuit below. The short-circuit's whole premise is that +# the full block is already in this session's context, and session start, /clear and +# compaction are precisely the events that destroy that context, so a recovery drain +# that printed a bare count would hand the recovering session a number with no key, +# no task and no resolve instruction. AGENTS.md section 3 makes the decision list +# part of every locked drain: recovery may trim bulk status tails, never the +# decisions. print_open_decisions_section() { - local snapshot=${1:-} open task key verb note line item_bytes=220 global_bytes=4000 - local output='' used=0 shown=0 omitted=0 bytes + local snapshot=${1:-} force_full=${2:-} open task key verb note line item_bytes=220 global_bytes=4000 + local output='' used=0 shown=0 omitted=0 bytes digest count marker prior + marker="$STATE/.open-decisions-presentation" if [ -n "$snapshot" ]; then open=$(scan_open_decisions_snapshot "$STATE" "$snapshot") || return 1 else open=$(scan_open_decisions_incremental "$STATE") || return 1 fi - [ -n "$open" ] || return 0 + if [ -z "$open" ]; then + rm -f "$marker" + return 0 + fi + count=$(printf '%s\n' "$open" | awk -F '\t' 'NF { n += 1 } END { print n + 0 }') || return 1 + digest=$(printf '%s' "$open" | open_decisions_digest) || return 1 + prior=$(cat "$marker" 2>/dev/null || true) + if [ "$force_full" != force ] && [ "$prior" = "$digest $count" ]; then + printf 'OPEN DECISIONS: unchanged, %d open\n' "$count" + return + fi while IFS=$(printf '\t') read -r task key verb note; do [ -n "$task" ] || continue @@ -175,6 +242,18 @@ EOF # depends on the busy worker writing a matching resolved line (contract: # bin/fm-send.sh header). printf "OPEN DECISIONS: close one by answering it: bin/fm-send.sh --resolve-key ''\n" || return 1 + OPEN_DECISIONS_PRESENTATION_PENDING="$digest $count" +} + +commit_open_decisions_presentation() { + local marker="$STATE/.open-decisions-presentation" tmp + [ "$PRESENTATION_COMMIT" -eq 1 ] || return 0 + [ -n "$OPEN_DECISIONS_PRESENTATION_PENDING" ] || return 0 + tmp="$marker.tmp.$$" + printf '%s\n' "$OPEN_DECISIONS_PRESENTATION_PENDING" > "$tmp" \ + || { rm -f "$tmp"; return 1; } + mv -f "$tmp" "$marker" || { rm -f "$tmp"; return 1; } + OPEN_DECISIONS_PRESENTATION_PENDING= } # Print the RECORD DIVERGENCE section: every captain call whose two records @@ -241,11 +320,16 @@ print_status_sections() { local snapshot=${1:-} fully_presented=${2:-} acknowledged if [ -z "$snapshot" ]; then snapshot=$(status_presentation_snapshot "$STATE") || return 1; fi [ -n "$snapshot" ] || return 0 - acknowledged=$(status_acknowledge_presented_snapshot "$STATE" "$snapshot" "$fully_presented") || return 1 + if [ "$PRESENTATION_COMMIT" -eq 1 ]; then + acknowledged=$(status_acknowledge_presented_snapshot "$STATE" "$snapshot" "$fully_presented") || return 1 + fi print_unread_status_section "$snapshot" || return 1 - print_open_decisions_section "$snapshot" || return 1 + print_open_decisions_section "$snapshot" "$OPEN_DECISIONS_FORCE" || return 1 print_record_divergence_section || return 1 - status_commit_presentation_snapshot "$STATE" "$acknowledged" + if [ "$PRESENTATION_COMMIT" -eq 1 ]; then + status_commit_presentation_snapshot "$STATE" "$acknowledged" || return 1 + fi + commit_open_decisions_presentation } print_status_presentation() { # [] @@ -264,6 +348,15 @@ print_status_presentation() { # [] return "$rc" } +print_compact_open_decisions() { + local lock="$STATE/.status-presentation-lock" rc=0 + fm_lock_acquire_wait "$lock" || return 1 + print_open_decisions_section '' "$OPEN_DECISIONS_FORCE" || rc=1 + if [ "$rc" -eq 0 ]; then commit_open_decisions_presentation || rc=1; fi + fm_lock_release "$lock" || rc=1 + return "$rc" +} + # shellcheck disable=SC2317,SC2329 # Invoked by trap handlers below. cleanup() { local status=$? @@ -342,12 +435,20 @@ if [ ! -s "$FM_WAKE_QUEUE" ]; then } RECOVERY_MARKER_TOKEN=$FM_RECOVERY_MARKER_TOKEN RECOVERY_ACK_REQUIRED=true + force_open_decisions_for_recovery + ;; + pending:handling:*|announced:handling:*) + RECOVERY_ACK_REQUIRED=true + force_open_decisions_for_recovery ;; - pending:handling:*|announced:handling:*) RECOVERY_ACK_REQUIRED=true ;; esac fm_lock_release "$FM_WAKE_QUEUE_LOCK" DRAIN_LOCK_HELD=false - (print_status_presentation) || echo "wake drain: status presentation failed; UNREAD STATUS, OPEN DECISIONS and RECORD DIVERGENCE may be incomplete" >&2 + if [ "$COMPACT" -eq 1 ]; then + (print_compact_open_decisions) || echo "wake drain: compact OPEN DECISIONS presentation failed" >&2 + else + (print_status_presentation) || echo "wake drain: status presentation failed; UNREAD STATUS, OPEN DECISIONS and RECORD DIVERGENCE may be incomplete" >&2 + fi if [ "$RECOVERY_ACK_REQUIRED" = true ]; then printf 'WAKE_ACK_REQUIRED: after handling completes run bin/fm-wake-drain.sh --ack-through 0 --recovery-generation %s\n' "${RECOVERY_MARKER_TOKEN##*:}" >&2 fi @@ -371,6 +472,13 @@ elif [ "${RECOVERY_MARKER_TOKEN%%:*}" = acked ]; then echo "wake drain: durable wakes could not enter a fresh recovery generation" >&2 exit 1 } +else + # An episode was already open before this drain: either the watcher-down path + # published it, or a previous handling turn never acknowledged it. Both are + # recovery, so the decisions are re-presented in full. A drain that opens its + # own generation for freshly queued rows (the two branches above) is the + # ordinary in-context loop and keeps the collapse. + force_open_decisions_for_recovery fi fm_recovery_marker_begin_handling "$RECOVERY_MARKER" || { echo "wake drain: durable wakes could not begin handling safely" >&2 @@ -399,6 +507,10 @@ DRAIN_LOCK_HELD=false printf 'WAKE_ACK_REQUIRED: after handling completes run bin/fm-wake-drain.sh --ack-through %s --recovery-generation %s\n' \ "$ACK_THROUGH" "${RECOVERY_MARKER_TOKEN##*:}" >&2 -(print_status_presentation "$RAW_ROWS") || echo "wake drain: status presentation failed; UNREAD STATUS, OPEN DECISIONS and RECORD DIVERGENCE may be incomplete" >&2 +if [ "$COMPACT" -eq 1 ]; then + (print_compact_open_decisions) || echo "wake drain: compact OPEN DECISIONS presentation failed" >&2 +else + (print_status_presentation "$RAW_ROWS") || echo "wake drain: status presentation failed; UNREAD STATUS, OPEN DECISIONS and RECORD DIVERGENCE may be incomplete" >&2 +fi assert_watcher_liveness exit 0 diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index f86833fe5d4..f1bd54e7f34 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -188,6 +188,24 @@ BUSY_TURN_MAX_SECS=${FM_BUSY_TURN_MAX_SECS:-3600} # These cases re-surface once for a recheck every PAUSE_RESURFACE_SECS - far # longer than the wedge threshold, but finite so a forgotten hold cannot rot invisibly. PAUSE_RESURFACE_SECS=${FM_PAUSE_RESURFACE_SECS:-$FM_PAUSE_RESURFACE_SECS_DEFAULT} +PAUSED_RESURFACE_KEYS='' +PAUSED_RESURFACE_ITEMS='' +PAUSED_RESURFACE_MARKERS='' +PAUSED_RESURFACE_COUNT=0 +PAUSED_RESURFACE_SHOWN=0 +PAUSED_RESURFACE_LIMIT=$FM_PAUSED_RESURFACE_BATCH_LIMIT_DEFAULT +# A missing or unreadable endpoint stays an immediate, same-cycle wake, but one +# backend restart (or several finished tasks whose .meta outlives their panes) +# makes it true of every window at once, and wake() ends the cycle - so +# per-window immediacy would cost one supervision turn per window for a single +# event. The cycle collects them here instead and delivers one fleet wake naming +# the windows before any batched routine recheck, so immediacy is preserved +# without amplifying churn. +MISSING_ENDPOINT_ITEMS='' +MISSING_ENDPOINT_MARKERS='' +MISSING_ENDPOINT_COUNT=0 +MISSING_ENDPOINT_SHOWN=0 +MISSING_ENDPOINT_LIMIT=$FM_ENDPOINT_BATCH_LIMIT_DEFAULT # Consecutive event-path failures (fm_backend_wait_transition returning 2 - # connect/subscribe failure) before the push fast-path is disabled for the rest # of this watcher process and the loop reverts to pure polling (report section @@ -319,12 +337,119 @@ FM_WEDGE_DEMAND_INSPECT_COUNT=${FM_WEDGE_DEMAND_INSPECT_COUNT:-3} # two cadences cannot drift apart; each caller owns its own marker and reason. # Returns without waking while either the absorb or the throttle is inside the # window; wake() itself exits the cycle, exactly as it does inline. -resurface_absorbed() { # - local win=$1 throttle=$2 age=$3 reason=$4 +# Fleet batching is an ATTENDED presentation: it collapses many due panes into one +# supervision turn for a human-driven primary. Away mode is daemon-owned and +# bin/fm-supervise-daemon.sh parses a stale reason as `stale: ()` +# to recover the window identity, so a batched fleet reason would classify under +# the literal string "paused fleet recheck" - recording a wedge marker for a window +# that does not exist while the real panes lose their per-pane pause markers. +# Under afk the caller therefore keeps the undecorated per-window wake it has +# always emitted, and only attended mode batches. +# /, when set by the caller, are committed ONLY +# where this pane's line is actually delivered, for the same reason the throttle is. +PAUSED_PRESENT_MARKER='' +PAUSED_PRESENT_VALUE='' + +resurface_absorbed() { # [batch] + local win=$1 throttle=$2 age=$3 reason=$4 mode=${5:-immediate} key + local present_marker=$PAUSED_PRESENT_MARKER present_value=$PAUSED_PRESENT_VALUE + PAUSED_PRESENT_MARKER='' + PAUSED_PRESENT_VALUE='' [ "$age" -ge "$PAUSE_RESURFACE_SECS" ] || return 0 [ "$(age_of "$throttle")" -ge "$PAUSE_RESURFACE_SECS" ] || return 0 # 999999 when no prior re-surface + if [ "$mode" = batch ] && ! afk_present; then + key=$(window_key "$win") + case "$PAUSED_RESURFACE_KEYS" in *"|$key|"*) return 0 ;; esac + PAUSED_RESURFACE_KEYS="$PAUSED_RESURFACE_KEYS|$key|" + PAUSED_RESURFACE_COUNT=$((PAUSED_RESURFACE_COUNT + 1)) + # The throttle marker is stamped ONLY for a pane whose reason actually makes + # it into the delivered batch. A pane past PAUSED_RESURFACE_LIMIT is counted + # but never named, so suppressing it for another PAUSE_RESURFACE_SECS would + # spend its bounded safety recheck on a wake that never mentioned it; leaving + # its marker untouched keeps it due, and it is named by the next cycle once + # the panes ahead of it are throttled. + if [ "$PAUSED_RESURFACE_SHOWN" -lt "$PAUSED_RESURFACE_LIMIT" ]; then + PAUSED_RESURFACE_ITEMS="${PAUSED_RESURFACE_ITEMS}${PAUSED_RESURFACE_ITEMS:+; }${reason#stale: }" + PAUSED_RESURFACE_SHOWN=$((PAUSED_RESURFACE_SHOWN + 1)) + PAUSED_RESURFACE_MARKERS="${PAUSED_RESURFACE_MARKERS}${PAUSED_RESURFACE_MARKERS:+ +}$throttle +$present_marker +$present_value" + fi + return 0 + fi fm_wake_append stale "$win" "$reason" || exit 1 date +%s > "$throttle" + [ -z "$present_marker" ] || printf '%s' "$present_value" > "$present_marker" + wake "$reason" +} + +flush_paused_resurfaces() { + local reason throttle present_marker present_value omitted + [ "$PAUSED_RESURFACE_COUNT" -gt 0 ] || return 0 + omitted=$((PAUSED_RESURFACE_COUNT - PAUSED_RESURFACE_SHOWN)) + reason="stale: paused fleet recheck (${PAUSED_RESURFACE_COUNT} due): $PAUSED_RESURFACE_ITEMS" + [ "$omitted" -eq 0 ] || reason="$reason; $omitted more omitted" + fm_wake_append stale paused-fleet "$reason" || exit 1 + while IFS= read -r throttle && IFS= read -r present_marker && IFS= read -r present_value; do + [ -n "$throttle" ] || continue + date +%s > "$throttle" + [ -z "$present_marker" ] || printf '%s' "$present_value" > "$present_marker" + done < marker, never the .stale- hash +# suppressor: overloading the hash suppressor would suppress a second +# disappearance forever (nothing rewrites the hash while the pane is busy), re-wake +# an already-surfaced stale pane on the next successful capture, and restart wedge +# aging on every flap. The marker is dropped by the first successful capture, so +# each distinct disappearance is reported exactly once. +# Same limit rule as the paused batch, under its OWN bound +# (FM_ENDPOINT_BATCH_LIMIT): tightening the paused batch must not silently truncate +# the endpoint wake, which reports a different failure to a different audience. A +# window past the limit is counted but not named, and its marker is left unwritten so +# the next cycle still reports it. +# Away mode is daemon-owned and needs the undecorated per-window identity to +# classify, so under afk this stays the immediate single-window wake it was. +record_missing_endpoint() { # + local win=$1 marker=$2 reason + if afk_present; then + reason="stale: $win (endpoint missing or unreadable)" + fm_wake_append stale "$win" "$reason" || exit 1 + : > "$marker" + wake "$reason" + return 0 + fi + MISSING_ENDPOINT_COUNT=$((MISSING_ENDPOINT_COUNT + 1)) + if [ "$MISSING_ENDPOINT_SHOWN" -lt "$MISSING_ENDPOINT_LIMIT" ]; then + MISSING_ENDPOINT_ITEMS="${MISSING_ENDPOINT_ITEMS}${MISSING_ENDPOINT_ITEMS:+; }$win" + MISSING_ENDPOINT_SHOWN=$((MISSING_ENDPOINT_SHOWN + 1)) + MISSING_ENDPOINT_MARKERS="${MISSING_ENDPOINT_MARKERS}${MISSING_ENDPOINT_MARKERS:+ +}$marker" + fi +} + +flush_missing_endpoints() { + local reason marker omitted + [ "$MISSING_ENDPOINT_SHOWN" -gt 0 ] || return 0 + if [ "$MISSING_ENDPOINT_COUNT" -eq 1 ]; then + reason="stale: $MISSING_ENDPOINT_ITEMS (endpoint missing or unreadable)" + else + omitted=$((MISSING_ENDPOINT_COUNT - MISSING_ENDPOINT_SHOWN)) + reason="stale: fleet endpoints missing or unreadable (${MISSING_ENDPOINT_COUNT}): $MISSING_ENDPOINT_ITEMS" + [ "$omitted" -eq 0 ] || reason="$reason; $omitted more omitted" + fi + fm_wake_append stale endpoint-fleet "$reason" || exit 1 + while IFS= read -r marker; do + [ -n "$marker" ] || continue + : > "$marker" + done < detail="paused, awaiting external" reason="paused ${age}s, awaiting external - declared pause, rechecked on a long cadence not a wedge; confirm the wait still holds" fi - resurface_absorbed "$win" "$STATE/.paused-resurfaced-$key" "$age" "stale: $win ($reason)" + PAUSED_PRESENT_MARKER="$STATE/.paused-presented-$key" + PAUSED_PRESENT_VALUE=$(last_status_line "$statusf") + resurface_absorbed "$win" "$STATE/.paused-resurfaced-$key" "$age" "stale: $win ($reason)" batch triage_log "absorbed stale ($detail, age ${age}s): $win" } @@ -481,7 +608,8 @@ busy_turn_bound_check() { # local key=$1 - rm -f "$STATE/.paused-$key" "$STATE/.paused-rechecked-$key" "$STATE/.paused-resurfaced-$key" + rm -f "$STATE/.paused-$key" "$STATE/.paused-rechecked-$key" \ + "$STATE/.paused-resurfaced-$key" "$STATE/.paused-presented-$key" } clear_pause_tracking() { # @@ -577,12 +705,74 @@ surface_nonterminal_stale() { # : > "$STATE/.paused-$key" date +%s > "$STATE/.paused-rechecked-$key" date +%s > "$STATE/.paused-resurfaced-$key" + printf '%s' "$last" > "$STATE/.paused-presented-$key" else - rm -f "$STATE/.paused-$key" "$STATE/.paused-rechecked-$key" "$STATE/.paused-resurfaced-$key" + clear_pause_state "$key" fi wake "stale: $win" } +# Records ONLY that this declaration was presented: the two markers +# attended_declared_wait_signal_is_absorbable reads. It deliberately does NOT arm +# .paused- or .paused-rechecked-. Delivering a wake is not an inspection: +# surface_nonterminal_stale may arm the cadence because pause_state_class has just +# spent its one live-agent gate for that key, and this path spends none. Arming here +# would let pause_state_class take its on-cadence short-circuit on the pane's first +# stale hash, so an ordinary crew still alive at an interactive permission or +# decision prompt would be absorbed onto the PAUSE_RESURFACE_SECS cadence instead of +# surfacing immediately - exactly the live decision gate that gate exists to protect. +record_declared_wait_presented() { # + local f=$1 task meta win key last + case "$f" in *.status) ;; *) return 0 ;; esac + task=$(basename "$f" .status) + meta="$STATE/$task.meta" + [ -f "$meta" ] && [ ! -L "$meta" ] || return 0 + win=$(fm_backend_target_of_meta "$meta" 2>/dev/null || true) + [ -n "$win" ] || return 0 + last=$(last_status_line "$f") + status_is_paused_or_captain_held "$last" || return 0 + key=$(window_key "$win") + date +%s > "$STATE/.paused-resurfaced-$key" + printf '%s' "$last" > "$STATE/.paused-presented-$key" +} + +# A repeat status or turn-end for an already presented declared wait is routine +# only while the exact durable declaration, endpoint readability, and bounded +# recheck window all still hold. Any uncertainty falls back to an immediate wake. +# A kind=secondmate task's .status is excluded through the same shared owner the +# provably-working absorb consults (signal_list_has_secondmate_status in +# fm-classify-lib.sh): that stream is the mate's routed-reply channel, so an +# unchanged repeat is still parent-directed content the supervisor must read. +# Absorption here stays scoped to ordinary crewmate paused-pane rechecks. +attended_declared_wait_signal_is_absorbable() { # ... + local f task meta win key last presented checked='' + attended_routine_status_absorb_enabled || return 1 + signal_reason_is_routine_nonterminal "$@" || return 1 + signal_reason_is_actionable "$@" && return 1 + signal_list_has_secondmate_status "$@" && return 1 + for f in "$@"; do + task=$(basename "$f") + task=${task%.status}; task=${task%.turn-ended} + case "$checked" in *"|$task|"*) continue ;; esac + checked="$checked|$task|" + [ -f "$STATE/$task.status" ] && [ -r "$STATE/$task.status" ] \ + && [ ! -L "$STATE/$task.status" ] || return 1 + last=$(last_status_line "$STATE/$task.status") + status_is_paused_or_captain_held "$last" || return 1 + meta="$STATE/$task.meta" + [ -f "$meta" ] && [ -r "$meta" ] && [ ! -L "$meta" ] || return 1 + win=$(fm_backend_target_of_meta "$meta" 2>/dev/null || true) + [ -n "$win" ] || return 1 + key=$(window_key "$win") + presented=$(cat "$STATE/.paused-presented-$key" 2>/dev/null || true) + [ "$presented" = "$last" ] || return 1 + [ "$(age_of "$STATE/.paused-resurfaced-$key")" -lt "$PAUSE_RESURFACE_SECS" ] || return 1 + fm_backend_capture "$(window_backend "$win")" "$win" 1 "$(window_label "$win")" \ + >/dev/null 2>&1 || return 1 + done + [ -n "$checked" ] +} + # Check and heartbeat cadence must survive actionable exits and restarts: the # watcher may be relaunched before in-memory counters reach their threshold on a # busy fleet. Persist the schedule as file mtimes instead. @@ -969,6 +1159,23 @@ resurface_after_downtime() { } while :; do + PAUSED_RESURFACE_KEYS='' + PAUSED_RESURFACE_ITEMS='' + PAUSED_RESURFACE_MARKERS='' + PAUSED_RESURFACE_COUNT=0 + PAUSED_RESURFACE_SHOWN=0 + MISSING_ENDPOINT_ITEMS='' + MISSING_ENDPOINT_MARKERS='' + MISSING_ENDPOINT_COUNT=0 + MISSING_ENDPOINT_SHOWN=0 + PAUSED_RESURFACE_LIMIT=${FM_PAUSED_RESURFACE_BATCH_LIMIT:-$FM_PAUSED_RESURFACE_BATCH_LIMIT_DEFAULT} + case "$PAUSED_RESURFACE_LIMIT" in + ''|*[!0-9]*|0) PAUSED_RESURFACE_LIMIT=$FM_PAUSED_RESURFACE_BATCH_LIMIT_DEFAULT ;; + esac + MISSING_ENDPOINT_LIMIT=${FM_ENDPOINT_BATCH_LIMIT:-$FM_ENDPOINT_BATCH_LIMIT_DEFAULT} + case "$MISSING_ENDPOINT_LIMIT" in + ''|*[!0-9]*|0) MISSING_ENDPOINT_LIMIT=$FM_ENDPOINT_BATCH_LIMIT_DEFAULT ;; + esac # Self-eviction: if the singleton lock no longer names this process, a second # watcher has taken over (e.g. a transient duplicate from a racy arm). Stand # down so the rightful singleton continues alone. The EXIT trap's release @@ -1131,7 +1338,9 @@ EOF # signal. A receipt failure leaves absorb_signal false, so the ordinary # durable wake path handles the event immediately. # shellcheck disable=SC2086 # $files is a space-separated status-path list (ids carry no spaces) - if ! afk_present && attended_signal_is_absorbable $files \ + if ! afk_present \ + && { attended_signal_is_absorbable $files \ + || attended_declared_wait_signal_is_absorbable $files; } \ && status_record_absorbed_signal "$STATE" $files; then absorb_signal=1 fi @@ -1146,6 +1355,7 @@ EOF [ -n "$sf" ] || continue printf '%s' "$sig" > "$sf" mark_surfaced "$f" + record_declared_wait_presented "$f" done </dev/null) || continue + sf="$STATE/.stale-$key" + emf="$STATE/.endpoint-missing-$key" + if ! tail40=$(fm_backend_capture "$(window_backend "$w")" "$w" 40 "$(window_label "$w")" 2>/dev/null); then + [ -e "$emf" ] || record_missing_endpoint "$w" "$emf" + continue + fi + rm -f "$emf" h=$(printf '%s' "$tail40" | hash_pane) hf="$STATE/.hash-$key" cf="$STATE/.count-$key" - sf="$STATE/.stale-$key" ssf="$STATE/.stale-since-$key" ewf="$STATE/.wedge-escalations-$key" pf="$STATE/.paused-$key" # flag: this key's stale is using the bounded pause cadence @@ -1238,12 +1453,23 @@ EOF clear_write_tracking "$key" triage_log "absorbed stale (provably working, overriding a stale captain-relevant status): $w" else - fm_wake_append stale "$w" "stale: $w" || exit 1 + terminal_status="$STATE/$(window_to_task "$w" "$STATE").status" + reason="stale: $w" + # A bare `stale: ` carries no verb and no path, so a + # downstream aggregator cannot tell a finished crew from a failed + # one and holds both for its full batch window. Naming the status + # file lets the reader resolve the actual last line, so failed:, + # blocked: and needs-decision: take the urgent bypass while done: + # and the rest stay routine. + if status_line_is_urgent "$(last_status_line "$terminal_status")"; then + reason="$reason ($terminal_status)" + fi + fm_wake_append stale "$w" "$reason" || exit 1 printf '%s' "$h" > "$sf" rm -f "$ssf" clear_write_tracking "$key" - mark_surfaced "$STATE/$(window_to_task "$w" "$STATE").status" - wake "stale: $w" + mark_surfaced "$terminal_status" + wake "$reason" fi elif [ -e "$ssf" ]; then # This exact hash was already overridden as provably-working (a @@ -1346,6 +1572,14 @@ EOF fi done < <(recorded_windows) + # Unreadable endpoints are the more urgent of the two batches, so they go + # first: an endpoint that vanished this cycle is still reported in this cycle. + flush_missing_endpoints + + # Declared waits are individually reconciled above, then presented once for + # the fleet so one cadence boundary costs one supervision turn and one drain. + flush_paused_resurfaces + # Heartbeat: the watcher runs a cheap fleet-scan at a regular cadence no matter # what. Time-based via .last-heartbeat mtime; interval doubles per consecutive # no-change heartbeat (idle fleet) up to HEARTBEAT_MAX, and resets on any diff --git a/docs/architecture.md b/docs/architecture.md index 2c2d92832d2..6e3a298fa78 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -45,6 +45,7 @@ Routine watcher polling, supervision no-ops, elapsed waiting time, and absorbed A declared external wait or verified captain-held transfer trades that silence for one bounded recheck per pause window, naming which human the wait is on, so neither a forgotten pause nor a forgotten hold can remain invisible indefinitely. Crew status files are append-only wake-event logs, not current-state fields. Because of that, a per-wake read of only the latest line can bury an earlier still-open `needs-decision`/`blocked` under later unrelated appends; `fm-wake-drain.sh` prints a separate, fleet-wide OPEN DECISIONS section on every presentation (including the empty-queue path session-start relies on), built through `fm-classify-lib.sh`'s cursor-backed incremental scan using the authoritative `status_open_decisions` fold semantics so the buried decision keeps surfacing until it is explicitly resolved while each presentation folds only new status-log appends. +A repeat presentation of an unchanged set collapses to an `OPEN DECISIONS: unchanged, N open` marker that refers back to the already-presented block, and a session-recovery drain, a drain handling an open watcher-down recovery episode, or a mechanical drain never spends that collapse; [`configuration.md`](configuration.md#attended-routine-status-absorption-configattended-routine-status-absorb--fm_attended_routine_status_absorb) owns that boundary. The drain coordinates that fold and its annotations through a locked fleet-wide snapshot whose `.status-presentation-cursor` manifest records each status file's identity and last-presented byte offset. A queued signal annotation prints every status line still unread at that cursor, while the fleet-wide UNREAD STATUS section prints `note:` lines, reserved-key pending-reply resolutions, and attended routine lines previously absorbed by bash exactly once on the next genuine presentation. An absorbed-status receipt binds the status-file identity and byte endpoint before the watcher advances its signal suppressor; cursor commit retires that receipt only after those bytes are printed, so a crash replays rather than loses the delayed status. diff --git a/docs/configuration.md b/docs/configuration.md index 128d3cec3b5..0411bfc58f8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -140,6 +140,39 @@ Any other value, and an unreadable or symlinked config file, disables absorption Away-mode triage is unaffected: while `state/.afk` exists the daemon owns every wake, and it reuses only the same recognized-shape boundary before distilling a digest entry. [`architecture.md`](architecture.md) owns how this fits the supervision loop and the delayed-presentation receipt. +An unchanged repeat status or turn-end from an ordinary crewmate already presented as `paused:` or `captain-held` is also absorbed while the exact durable declaration, readable endpoint, and bounded pause-recheck window still hold. +The first declaration, a changed terminal verb, a missing or unreadable endpoint, and a lapsed bounded recheck always wake immediately. +A `kind=secondmate` task's `.status` is never absorbed by either absorber: that stream is the mate's routed-reply channel, so even an unchanged repeat is parent-directed content the supervisor must read. +A mate's bare `.turn-ended` ping is not covered by that carve-out and still uses the ordinary provably-working absorb. + +A compact-recovery drain (`bin/fm-wake-drain.sh --compact`) always prints the full bounded OPEN DECISIONS block rather than the `OPEN DECISIONS: unchanged, N open` collapse. +A drain that handles an open watcher-down recovery episode - the downtime marker `bin/fm-watch-arm.sh` publishes, or a handling episode a previous turn never acknowledged - prints the full block for the same reason: the decisions were last presented to a session that is no longer supervising them, and a decision-only recovery would otherwise re-surface as a bare count with no key, task, or resolve instruction. +The collapse's premise is that the full block is already in this session's context, and compaction is exactly the event that destroys it; compaction may trim bulk status tails, never the decision list. +For the same reason the collapse is only ever spent by a drain a human actually reads: the away-mode daemon drains with `--no-presentation-commit` because it consumes the wake rows mechanically and discards the presented text, so the block survives intact for whoever reads it next, and `bin/fm-afk-return.sh` drains in recovery mode so a captain returning from away mode is handed the decisions in full. +`--no-presentation-commit` withholds the UNREAD STATUS cursor as well: exact-once means presented once to a reader, so a mechanical drain may read status bytes but never advances the cursor or marks lines absorbed. + +A terminal stale wake names its task's status file when that status is `failed:`, `blocked:` or `needs-decision:`, so an aggregator can resolve the verb and take the urgent bypass instead of holding the failure for a batch window. +`done:` and every other terminal status keeps the bare `stale: ` identity, and `paused:` and `working:` are never urgent. +In attended mode a cadence boundary presents every due declared wait in one fleet stale wake, bounded by `FM_PAUSED_RESURFACE_BATCH_LIMIT`. +Away mode is unaffected: while `state/.afk` exists the daemon owns triage and still receives the undecorated per-window stale identity it parses, for declared waits and for missing or unreadable endpoints alike. +Only a pane actually named in that wake has its bounded recheck throttled; a pane counted past the limit stays due and is named by a later cycle, so no pane loses its safety recheck to a wake that never mentioned it. +Missing or unreadable endpoints found in one attended watcher cycle are likewise collected into one immediate fleet wake naming those windows, so a backend restart costs one supervision turn rather than one per window while detection stays in the same cycle. +Each window's disappearance is tracked by its own marker, dropped by the first successful capture, so a flapping endpoint reports each distinct disappearance exactly once without disturbing the pane's recorded stale hash or its wedge timer. +That list has its own bound, `FM_ENDPOINT_BATCH_LIMIT`, so tightening the paused batch never silently truncates it. + +## Pi watcher wake batching (config/wake-batch-seconds) + +Pi and pi-signed primary homes aggregate routine watcher closes before delivering one `FIRSTMATE WATCHER WAKE` follow-up. +The default window is 60 seconds. +Write one positive integer number of seconds to gitignored `config/wake-batch-seconds` to change it for that home. +`FM_WAKE_BATCH_SECONDS` is the process-local override used by tests and specialized launches. +Identical status paths and backend endpoints are deduplicated, the rendered list is bounded, and one delivered batch requires one drain and one acknowledgement. +Urgent details are rendered ahead of routine ones, so the bound can only omit routine wakes and never the failure that triggered the flush. +A batch does not outlive its CURRENT watcher arm when that arm is not replaced: an actionable close starts a successor immediately and the batch keeps aggregating across that rotation, handing ownership to the successor each time, but a non-actionable close hands off to a bounded retry, so the batch is flushed there rather than waiting out a window under a cycle that may not come back. +An arm only exits after the watcher it waited on has exited, so that arm-end flush does not attempt the handling confirmation: there is nothing left to confirm, and confirming would report a watcher failure that never happened. A timer-driven flush still confirms. +The durable wake queue and the recovery marker remain the only records of what was delivered and acknowledged, so a crash between that flush and its confirmation re-presents on the next drain instead of losing the batch. +`failed:`, `blocked:`, `needs-decision:`, lost-lock, and watcher-failure wakes bypass the delay and flush any pending routine batch immediately. + ## Gate defaults (.no-mistakes.yaml) The tracked `.no-mistakes.yaml` sets `test.evidence.store_in_repo: true` and pins `commands.lint` to `bin/fm-lint.sh` so local lint matches CI. @@ -605,6 +638,7 @@ FM_TRACE_CONTEXT= # optional trace-context override; see "Trace context pr HERDR_SESSION=default # herdr-only: named session for normal backend ops; not enough for destructive cleanup (docs/herdr-backend.md) FM_BACKEND_HERDR_SUBMIT_POLLS=6 # herdr-only: agent-state samples spread across each Enter attempt's budget when confirming a submit (docs/herdr-backend.md "Current transport behavior") FM_BACKEND_HERDR_SUBMIT_MIN_SLEEP=0.6 # herdr-only: minimum per-Enter confirmation budget before polling agent-state after an idle baseline +FM_BACKEND_HERDR_UNKNOWN_RECHECKS=3 # herdr-only: keypress-free re-observations of an inconclusive post-Enter surface before a submit is reported unknown (docs/herdr-backend.md "Current transport behavior") FM_ZELLIJ_SESSION=firstmate # zellij-only: named session for normal backend ops and test isolation (docs/zellij-backend.md) CMUX_SOCKET_PASSWORD= # cmux-only: socket password fallback when config/cmux-socket-password is absent (docs/cmux-backend.md) FM_SESSION_START_STATUS_TAIL=5 # state/*.status lines printed per task in the session-start digest; each line is capped by bin/fm-line-cap-lib.sh @@ -663,6 +697,10 @@ FM_WATCH_CYCLE_LOG_KEEP_LINES=1000 # newest complete lifecycle rows considered FM_WATCHER_STALE_GRACE=300 # defaults to FM_GUARD_GRACE; seconds a live watcher lock may have a stale beacon before re-arm errors FM_SIGNAL_GRACE=30 # seconds to coalesce nearby status and turn-end signals into one wake FM_ATTENDED_ROUTINE_STATUS_ABSORB=on # attended routine-status absorption override; see "Attended routine status absorption" +FM_WAKE_BATCH_SECONDS=60 # Pi watcher follow-up aggregation window; config/wake-batch-seconds is the home-local owner +FM_WAKE_BATCH_LIMIT=20 # maximum distinct routine watcher items rendered in one Pi follow-up +FM_PAUSED_RESURFACE_BATCH_LIMIT=20 # maximum due paused/captain-held pane details in one fleet stale wake +FM_ENDPOINT_BATCH_LIMIT=20 # maximum missing or unreadable endpoint windows named in one fleet stale wake FM_CAPTAIN_RE='done:|needs-decision:|blocked:|failed:|PR ready|checks green|ready in branch|merged' # captain-relevant status regex; nonterminal progress verbs remain excluded even when their prose matches FM_CLASSIFY_PAUSED_VERB=paused # leading status verb for a declared external wait; excluded from FM_CAPTAIN_RE and distinct from blocked FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane escalates; stale panes whose crew is not provably working surface immediately unless they declare the pause verb diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index 029726f8bef..1559235ac85 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -216,7 +216,8 @@ On an idle or done native baseline, submit confirmation first waits for `working If native status stays idle, the shared composer verdict is the next positive signal: a cleared composer is delivery, and proven pending text retries Enter. After the retry budget, `fm_composer_queued_enter_verdict` treats proven pending text plus a generating busy signal as a queued delivered Enter, and keeps an idle pending composer as a genuine swallow. On an already active or unreadable baseline, the adapter falls back to conservative composer clearance, with a pre-Enter rendered-footer transition when that baseline is unavailable. -A fully unreadable target stops retrying and reports unknown. +An inconclusive post-Enter surface is re-observed by a bounded recheck (`FM_BACKEND_HERDR_UNKNOWN_RECHECKS`, default 3) that sends no further keypress: a later empty composer or idle-baseline busy transition confirms the submit, and a later proven-pending composer authorizes the ordinary bounded Enter retry. +Only a target still unreadable after that recheck stops retrying and reports unknown, so persistent unreadability never receives a blind keypress and the missing-endpoint safety boundary holds. blocked is not treated as a queued-Enter busy signal, so a Cursor pane that reports blocked in every state does not receive that conversion. Some harnesses never present a legibly idle native baseline at all, so the composer fallback is their only path. diff --git a/docs/scripts.md b/docs/scripts.md index a0f811d179e..2fa1d256057 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -37,7 +37,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-test-isolation-proof.sh` | Concurrent isolation proof and proven-isolated candidate set owner | | `fm-ensure-agents-md.sh` | Ensure a project's real `AGENTS.md`, its `CLAUDE.md` `@AGENTS.md` pointer, and the canonical self-governance section | | `fm-guard.sh` | Warn on primary-checkout tangles, pending queued wakes, and unhealthy supervision | -| `fm-primary-scope-lib.sh` | Shared marker-or-plain-checkout primary-home predicate for tracked hooks | +| `fm-primary-scope-lib.sh` | Shared marker-or-plain-checkout primary-home predicate and registered-crew-worktree digest suppression for tracked hooks | | `fm-session-lock-lib.sh` | Shared session-lock harness identity (ancestry walk and holder liveness) for fm-lock.sh and the Claude Stop auto-arm | | `fm-claude-stop-autoarm.sh` | Claude Stop `asyncRewake` hook owning tokenless watcher continuity with single-flight exit-2 rewake (docs/watcher-continuity.md) | | `fm-turnend-guard.sh` | Shared primary turn-end guard predicate so no turn ends blind (docs/turnend-guard.md) | @@ -46,7 +46,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-arm-pretool-check.sh` | Stable PreToolUse transport for the watcher-arm command policy (docs/arm-pretool-check.md) | | `fm-arm-command-policy.mjs` | Semantic owner of the watcher-arm PreToolUse policy (docs/arm-pretool-check.md) | | `fm-subagent-pretool-check.sh` | Primary-home delegation-shape PreToolUse guard (docs/subagent-guard.md) | -| `fm-supervision-instructions.sh` | Render the session-start primary-harness supervision block or the one-line repair instruction | +| `fm-supervision-instructions.sh` | Render the session-start primary-harness supervision block, the one-line repair instruction, or the compact-recovery state and next-instruction lines | | `fm-home-seed.sh` | Transactionally provision a local secondmate home and maintain `data/secondmates.md` | | `fm-remote-home-seed.sh` | Register and provision a whole secondmate home on an SSH-reachable host | | `fm-remote-readiness-lib.sh` | Shared remote second-mate readiness gate: check and, when needed, repair then re-check through `fm-remote-doctor.sh` | diff --git a/docs/sessionstart-nudge.md b/docs/sessionstart-nudge.md index 4e4b11c18dd..f3f4bd90ba1 100644 --- a/docs/sessionstart-nudge.md +++ b/docs/sessionstart-nudge.md @@ -24,7 +24,8 @@ It takes `--source ` when the adapter knows the source natively, and other | Source | Action | Why | | --- | --- | --- | | `startup`, `new` | Full digest | This is a true session start that has not taken the helm; Pi CLI continuations are refined to `resume` by the adapter before reaching this boundary. | -| `clear`, `compact` | `--reemit` after a proven complete startup, otherwise full digest | This process normally has the helm and lost only its context, but an earlier hook may have been truncated after acquiring the lock. | +| `clear` | `--reemit` after a proven complete startup, otherwise full digest | This process normally has the helm and lost only its context, but an earlier hook may have been truncated after acquiring the lock. | +| `compact` | Compact recovery digest after a proven complete startup, otherwise full digest | Pi compaction needs current supervision ownership and actionable identities, not repeated status tails and unchanged context files. | | `resume`, `reload`, `fork` | Delegate to the nudge wrapper | Prior context is restored, so re-running is redundant when the lock is still ours and an instruction is enough when a new process resumed an old session. | | unreadable or unrecognized | Full digest | Taking the helm redundantly is cheap and idempotent; not taking it is the bug this tier exists to fix. | @@ -33,7 +34,18 @@ Compaction is covered where a tracked adapter delivers that source because a com Current harness ownership of the lock and its matching `state/.session-start-complete` record together are the idempotency interlock for the whole scheme. The full digest clears that completion record after acquiring the lock and republishes the lock owner's pid only after every stage completes, so `clear` or `compact` cannot skip startup sweeps after a truncated run. -`bin/fm-lock.sh` already treats a lock this session's own harness holds as its own, so a proven `clear` or `compact` re-emit re-verifies ownership and proceeds, while a lock another live session took meanwhile still produces the ordinary read-only digest. +`bin/fm-lock.sh` already treats a lock this session's own harness holds as its own, so a proven `clear` re-emit or `compact` recovery re-verifies ownership and proceeds, while a lock another live session took meanwhile still produces read-only guidance. +The compact recovery digest is bounded to supervision ownership, the actionable queue and open decisions, active task identities, and the next supervision instruction. +Away-mode and X-mode state ride in that bound as two short lines under lock and watcher ownership, printed by `bin/fm-supervision-instructions.sh --state-lines`, because both change who owns supervision and what a wake means; a recovering session that read them as attended while the daemon owned triage would cross an AGENTS.md section 8 boundary. +That digest carries no protocol snippet, so `bin/fm-supervision-instructions.sh --next-line` is self-contained for every harness: the drain and acknowledgement steps, the condition, the exact command where the model owns the next cycle, and the path of the owning `docs/supervision-protocols/` document. +Under away mode that line instead routes the recovering session to the daemon: it names the `/afk` action, the `state/.afk` condition, the `bin/fm-supervise-daemon.sh` owner, and explicitly forbids the attended drain and acknowledgement, because the daemon triages the same durable queue. +It names that document rather than inlining it, because a compacted session cannot follow a pointer into context it has lost, and inlined protocol would defeat the compaction. + +When the tracked Firstmate checkout is itself a registered crew worktree, both session-start wrappers print only `crew worktree - digest suppressed`. +Registration is proven by an exact `worktree=` match in the primary checkout's task metadata; unmarked linked worktrees remain silent, and marked secondmate homes remain eligible primaries. +`bin/fm-session-start.sh` runs that git-backed lookup under its own short bound outside the timed digest child, and any timeout or failure falls through to the ordinary digest, because a redundant startup is cheaper than a silent one. +That lookup is scoped to the primary checkout's own default home, `/state`, because a crew worktree carries no registrations of its own and an ordinary crew session inherits no `FM_HOME`. +A crew worktree registered by some other home is therefore not suppressed from that home's registrations; each home runs its own session start against its own, and cross-home discovery is deliberately not part of this scheme. On a run-tier harness the nudge cannot also fire: `resume`, `reload`, and `fork` are the only sources routed to it, and on those its own ancestry check stays silent whenever this process already holds the lock. `bin/fm-session-start.sh --reemit` owns which work a re-emit skips, its true-start AGENTS.md baseline, and its supported stale-instruction refresh pairs; its header is the single owner of those mechanics. diff --git a/docs/supervision-protocols/pi.md b/docs/supervision-protocols/pi.md index 5cdcaed7b08..1956501bc78 100644 --- a/docs/supervision-protocols/pi.md +++ b/docs/supervision-protocols/pi.md @@ -12,6 +12,9 @@ When this session owns supervision and away mode is not active: 6. Ordinary same-process session replacement (`/new`, `/resume`, `/fork`, reload) retires only the prior generation; call `fm_watch_arm_pi` once for the first cycle of the replacement session without restarting Pi. The generation-owner contract lives in `.pi/extensions/fm-primary-pi-watch.ts`. 7. After an actionable child close, the extension rechecks session-lock ownership and verifies one successor before it delivers the follow-up wake; its bounded fallback is defined in `docs/watcher-continuity.md`. + Routine closes are aggregated for the home-configured `config/wake-batch-seconds` window, default 60 seconds, with identical status paths and endpoints deduplicated into one bounded follow-up. + Failed, blocked, needs-decision, lost-lock, and watcher-failure classes bypass the delay and flush immediately. + One delivered batch is one handling turn: drain once, handle the bounded list, then run the one acknowledgement printed by that drain. 8. Ordinary work, turn completion, and ordinary signal, stale, check, heartbeat, or other wake handling: do not call `fm_watch_arm_pi` again because continuity is extension-owned rather than model-memory-owned. 9. An unexpected child close enters bounded exponential retry, and an exhausted retry or lost session lock is surfaced as a watcher failure instead of disappearing. 10. Missing, failed, or unhealthy cycle only: if a later notification explicitly reports one of those repair conditions, drain queued wakes, inspect the failure text, call `fm_watch_arm_pi`, and restart the selected Pi-family executable with both extensions loaded if needed. diff --git a/docs/turnend-guard.md b/docs/turnend-guard.md index de9b5ed922e..1ab4473d08b 100644 --- a/docs/turnend-guard.md +++ b/docs/turnend-guard.md @@ -98,6 +98,9 @@ Their adapters fail open at the hook boundary to protect the user session but sc The generated prompts use the canonical `turn-end-guard` kind after the U+2063 `FIRSTMATE_OP: ` prefix, so Ahoy does not treat them as captain messages. Each passive adapter owns a loop latch. Pi keeps the latch across internal tool turns and clears it only when the generated follow-up settles or delivery fails. +On `session_compact`, the Pi guard injects the compact recovery digest rather than the full startup re-emit. +That digest contains only lock and watcher ownership, the actionable queue and open decisions, active task identities, and the exact next supervision instruction. +It omits status tails and unchanged context files; unread routine status remains durable for the next ordinary drain instead of being acknowledged by the compact view. OpenCode's forced follow-up is supported for persistent TUI sessions and remains fail-open in headless `opencode run`. Grok makes exactly one typed capability decision from each running Stop payload. diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index df16f486532..605809953b9 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -30,6 +30,8 @@ If the unready arm does not retire within that bound, the adapter keeps ownershi When that retained arm later closes, its actual close is classified as a new supervised event without replaying the earlier fallback. After the configured retry bound is exhausted, it delivers the original wake with a typed continuity-restoration failure even if every successor arm hung without reporting readiness. This is deliberate Option B ordering: the fleet is protected before the model handles the wake whenever restoration succeeds, but the model is never left blind when it does not. +Pi's follow-up aggregation leaves that ordering intact: a successor is still started and verified per actionable close, and only the delivery of routine wakes is deferred into one bounded follow-up, with urgent classes flushing immediately. +The handling confirmation moves with that delivery - it runs once per delivered batch against the batch's latest recovery, and an arm-end flush skips it because the arm outlives its own watcher and there is nothing left to confirm; [`configuration.md`](configuration.md#pi-watcher-wake-batching-configwake-batch-seconds) owns the window, its bounds, and that flush. Claude's Stop hook starts the successor arm at the next Stop after the handling turn, rather than before notification as Pi and OpenCode do. The durable wake queue preserves actionable events during the residual active-turn window, and the bounded turn-end guard enforces recovery at Stop when no watcher is live and no auto-arm claim is still deciding, so a leftover claim whose own decision already finished cannot suppress it ([`turnend-guard.md`](turnend-guard.md#harness-integrations) owns that boundary). diff --git a/tests/fm-backend-herdr.test.sh b/tests/fm-backend-herdr.test.sh index dc1be58f9c5..3e307e74efa 100755 --- a/tests/fm-backend-herdr.test.sh +++ b/tests/fm-backend-herdr.test.sh @@ -3799,6 +3799,65 @@ test_send_text_submit_unknown_on_composer_capture_failure() { pass "fm_backend_herdr_send_text_submit: an unreadable composer stops Enter retries after native status stays idle" } +test_send_text_submit_rechecks_transient_unknown_before_retrying_enter() { + local dir log out + dir="$TMP_ROOT/submit-transient-unknown"; mkdir -p "$dir"; log="$dir/enters"; : > "$log" + out=$(FM_ENTER_LOG="$log" FM_CASE_DIR="$dir" bash -c ' + . "$0/bin/backends/herdr.sh" + fm_backend_herdr_parse_target() { FM_BACKEND_HERDR_SESSION=default; FM_BACKEND_HERDR_PANE=w1:p2; } + fm_backend_herdr_send_literal() { return 0; } + fm_backend_herdr_send_key() { printf "enter\n" >> "$FM_ENTER_LOG"; return 0; } + fm_backend_herdr_agent_status_raw() { printf idle; } + fm_backend_herdr_classify_submit_agent_status() { printf idle; } + fm_backend_herdr_rendered_busy_state() { printf idle; } + fm_backend_herdr_submit_confirm_budget() { printf 0.01; } + fm_backend_herdr_wait_for_working() { + calls=$(cat "$FM_CASE_DIR/waits" 2>/dev/null || echo 0); calls=$((calls + 1)); echo "$calls" > "$FM_CASE_DIR/waits" + if [ "$calls" -eq 1 ]; then printf unknown; else printf busy; fi + } + fm_backend_herdr_composer_state() { + calls=$(cat "$FM_CASE_DIR/composers" 2>/dev/null || echo 0); calls=$((calls + 1)); echo "$calls" > "$FM_CASE_DIR/composers" + if [ "$calls" -eq 1 ]; then printf unknown; else printf pending; fi + } + sleep() { :; } + fm_backend_herdr_send_text_submit default:w1:p2 "hello" 3 0.01 0 + ' "$ROOT") + [ "$out" = empty ] || fail "a transient unknown submit was not recovered, got '$out'" + [ "$(wc -l < "$log" | tr -d ' ')" -eq 2 ] \ + || fail "a proven-pending transient unknown did not receive exactly one bounded Enter retry" + pass "fm_backend_herdr_send_text_submit: transient unknown is observed until pending, then retried once and confirmed" +} + +test_send_text_submit_idle_branch_trusts_proven_pending_recheck() { + local dir log out + dir="$TMP_ROOT/submit-idle-proven-pending"; mkdir -p "$dir"; log="$dir/enters"; : > "$log" + out=$(FM_ENTER_LOG="$log" FM_CASE_DIR="$dir" bash -c ' + . "$0/bin/backends/herdr.sh" + fm_backend_herdr_parse_target() { FM_BACKEND_HERDR_SESSION=default; FM_BACKEND_HERDR_PANE=w1:p2; } + fm_backend_herdr_send_literal() { return 0; } + fm_backend_herdr_send_key() { printf "enter\n" >> "$FM_ENTER_LOG"; return 0; } + fm_backend_herdr_agent_status_raw() { printf idle; } + fm_backend_herdr_classify_submit_agent_status() { printf idle; } + fm_backend_herdr_rendered_busy_state() { printf idle; } + fm_backend_herdr_submit_confirm_budget() { printf 0.01; } + fm_backend_herdr_wait_for_working() { + calls=$(cat "$FM_CASE_DIR/waits" 2>/dev/null || echo 0); calls=$((calls + 1)); echo "$calls" > "$FM_CASE_DIR/waits" + if [ "$calls" -eq 1 ]; then printf unknown; else printf busy; fi + } + fm_backend_herdr_composer_state() { + calls=$(cat "$FM_CASE_DIR/composers" 2>/dev/null || echo 0); calls=$((calls + 1)); echo "$calls" > "$FM_CASE_DIR/composers" + if [ "$calls" -eq 1 ]; then printf pending; else printf unknown; fi + } + sleep() { :; } + fm_backend_herdr_send_text_submit default:w1:p2 "hello" 3 0.01 0 + ' "$ROOT") + [ "$out" = empty ] \ + || fail "a later transient unknown discarded the recheck's proven-pending verdict, got '$out'" + [ "$(wc -l < "$log" | tr -d " ")" -eq 2 ] \ + || fail "the proven-pending recheck did not authorize exactly one bounded Enter retry" + pass "fm_backend_herdr_send_text_submit: an idle-baseline proven-pending recheck is not overwritten by a later unknown read" +} + # --- fm-backend.sh dispatch wiring ------------------------------------------- test_dispatch_routes_herdr_backend() { @@ -4561,6 +4620,8 @@ test_send_text_submit_slow_transition_within_one_enter_needs_no_extra_enter test_send_text_submit_send_failed test_send_text_submit_unknown_on_capture_failure test_send_text_submit_unknown_on_composer_capture_failure +test_send_text_submit_rechecks_transient_unknown_before_retrying_enter +test_send_text_submit_idle_branch_trusts_proven_pending_recheck test_dispatch_routes_herdr_backend test_dispatch_busy_state_unknown_for_tmux test_dispatch_composer_state_routes_by_backend diff --git a/tests/fm-pi-watch-extension.test.sh b/tests/fm-pi-watch-extension.test.sh index fb473ee0343..0274b6310d7 100755 --- a/tests/fm-pi-watch-extension.test.sh +++ b/tests/fm-pi-watch-extension.test.sh @@ -11,6 +11,9 @@ EXT="$ROOT/.pi/extensions/fm-primary-pi-watch.ts" # from a clean checkout with no tracked .opencode/package.json. The warning is # unrelated to plugin output, which the assertions intentionally require empty. export NODE_NO_WARNINGS=1 +# Production defaults to a 60-second aggregation window; fixtures keep the same +# behavior with a one-second window so delivery assertions stay fast. +export FM_WAKE_BATCH_SECONDS=1 # One owner for the readiness budget every unready-successor test below spends # on purpose. Both plugins start a successor arm through a login shell and @@ -394,6 +397,7 @@ for (let i = 0; i < 250; i += 1) { if (rows.length >= 2 && deliveryStarted) break; await new Promise((resolve) => setTimeout(resolve, 10)); } + const rows = readFileSync(process.env.FM_ARM_LOG, "utf8").trim().split("\n"); const armRows = rows.filter((row) => row.startsWith("arm=")); if (armRows.length !== 2) throw new Error(`expected one successor arm, got ${armRows.length}: ${rows.join(" | ")}`); @@ -422,6 +426,534 @@ EOF pass "Pi actionable close starts one successor before wake delivery settles" } +test_pi_batches_and_dedupes_routine_wakes() { + local repo home plugin log stop out status + repo="$TMP_ROOT/pi-wake-batch-root"; home="$TMP_ROOT/pi-wake-batch-home" + log="$TMP_ROOT/pi-wake-batch.log"; stop="$TMP_ROOT/pi-wake-batch.stop" + mkdir -p "$repo/bin" "$home/state" "$home/config" + printf 'working: routine a\n' > "$home/state/a.status" + printf 'working: routine b\n' > "$home/state/b.status" + install_pi_watch_extension_fixture "$repo" + plugin="$repo/.pi/extensions/fm-primary-pi-watch.ts" + cat > "$repo/bin/fm-watch-arm.sh" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --handling-delivered ]; then + printf 'confirmed\n' >> "${FM_ARM_LOG:?}" + exit 0 +fi +printf 'arm\n' >> "${FM_ARM_LOG:?}" +count=$(grep -c '^arm$' "$FM_ARM_LOG") +printf 'watcher: started pid=%s (beacon fresh) recovery-generation=batch-%s\n' "$$" "$count" +case "$count" in + 1) printf 'signal: %s/state/a.status\n' "$FM_HOME" ;; + 2) sleep 0.1; printf 'signal: %s/state/a.status\n' "$FM_HOME" ;; + 3) sleep 0.1; printf 'signal: %s/state/b.status\n' "$FM_HOME" ;; + *) trap 'exit 0' TERM INT; while [ ! -e "$FM_STOP_FILE" ]; do sleep 0.02; done ;; +esac +SH + chmod +x "$repo/bin/fm-watch-arm.sh" + out=$(PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" FM_ARM_LOG="$log" FM_STOP_FILE="$stop" node --input-type=module 2>&1 <<'EOF' +import { readFileSync, writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +let tool = null; +const deliveries = []; +const pi = { + on() {}, registerCommand() {}, + registerTool(candidate) { if (candidate.name === "fm_watch_arm_pi") tool = candidate; }, + sendUserMessage: async (content) => { deliveries.push(content); }, +}; +writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); +const mod = await import(pathToFileURL(process.env.PLUGIN).href); +mod.default(pi); +await tool.execute("batch", {}, undefined, undefined, {}); +for (let i = 0; i < 400 && deliveries.length === 0; i += 1) await new Promise((resolve) => setTimeout(resolve, 10)); +if (deliveries.length !== 1) throw new Error(`expected one batched follow-up, got ${deliveries.length}`); +const message = deliveries[0]; +if (!message.includes("batched 2 watcher wakes")) throw new Error(`batch did not contain two distinct wakes: ${message}`); +if ((message.match(/a\.status/g) ?? []).length !== 1) throw new Error(`duplicate status path was not deduped: ${message}`); +if (!message.includes("b.status")) throw new Error(`second status path was omitted: ${message}`); +const rows = readFileSync(process.env.FM_ARM_LOG, "utf8").trim().split("\n"); +if (rows.filter((row) => row === "confirmed").length !== 1) throw new Error(`batch was not confirmed exactly once: ${rows.join(" | ")}`); +writeFileSync(process.env.FM_STOP_FILE, "stop\n"); +EOF + ) + status=$? + expect_code 0 "$status" "Pi routine wake batching" + [ -z "$out" ] || fail "Pi routine wake batching printed output: $out" + pass "Pi watcher batches routine wakes, dedupes repeated status paths, and confirms once" +} + +test_pi_urgent_status_bypasses_batch_window() { + local repo home plugin log stop out status + repo="$TMP_ROOT/pi-wake-urgent-root"; home="$TMP_ROOT/pi-wake-urgent-home" + log="$TMP_ROOT/pi-wake-urgent.log"; stop="$TMP_ROOT/pi-wake-urgent.stop" + mkdir -p "$repo/bin" "$home/state" "$home/config" + printf 'blocked [key=route]: captain decision required\n' > "$home/state/urgent.status" + install_pi_watch_extension_fixture "$repo" + plugin="$repo/.pi/extensions/fm-primary-pi-watch.ts" + cat > "$repo/bin/fm-watch-arm.sh" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --handling-delivered ]; then exit 0; fi +printf 'arm\n' >> "${FM_ARM_LOG:?}" +count=$(grep -c '^arm$' "$FM_ARM_LOG") +printf 'watcher: started pid=%s (beacon fresh) recovery-generation=urgent-%s\n' "$$" "$count" +if [ "$count" -eq 1 ]; then + printf 'signal: %s/state/urgent.status\n' "$FM_HOME" +else + trap 'exit 0' TERM INT + while [ ! -e "$FM_STOP_FILE" ]; do sleep 0.02; done +fi +SH + chmod +x "$repo/bin/fm-watch-arm.sh" + out=$(PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" FM_ARM_LOG="$log" \ + FM_STOP_FILE="$stop" FM_WAKE_BATCH_SECONDS=30 node --input-type=module 2>&1 <<'EOF' +import { writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +let tool = null; +const deliveries = []; +const pi = { + on() {}, registerCommand() {}, + registerTool(candidate) { if (candidate.name === "fm_watch_arm_pi") tool = candidate; }, + sendUserMessage: async (content) => { deliveries.push(content); }, +}; +writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); +const mod = await import(pathToFileURL(process.env.PLUGIN).href); +mod.default(pi); +await tool.execute("urgent", {}, undefined, undefined, {}); +for (let i = 0; i < 50 && deliveries.length === 0; i += 1) await new Promise((resolve) => setTimeout(resolve, 10)); +if (deliveries.length !== 1) throw new Error("blocked status waited for the 30-second routine batch window"); +if (!deliveries[0].includes("urgent.status")) throw new Error(`urgent wake path missing: ${deliveries[0]}`); +writeFileSync(process.env.FM_STOP_FILE, "stop\n"); +EOF + ) + status=$? + expect_code 0 "$status" "Pi urgent wake bypass" + [ -z "$out" ] || fail "Pi urgent wake bypass printed output: $out" + pass "Pi blocked status bypasses the configured routine wake window" +} + +test_pi_batch_keeps_both_reasons_for_one_endpoint() { + local repo home plugin log stop out status + repo="$TMP_ROOT/pi-wake-identity-root"; home="$TMP_ROOT/pi-wake-identity-home" + log="$TMP_ROOT/pi-wake-identity.log"; stop="$TMP_ROOT/pi-wake-identity.stop" + mkdir -p "$repo/bin" "$home/state" "$home/config" + install_pi_watch_extension_fixture "$repo" + plugin="$repo/.pi/extensions/fm-primary-pi-watch.ts" + cat > "$repo/bin/fm-watch-arm.sh" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --handling-delivered ]; then exit 0; fi +printf 'arm\n' >> "${FM_ARM_LOG:?}" +count=$(grep -c '^arm$' "$FM_ARM_LOG") +printf 'watcher: started pid=%s (beacon fresh) recovery-generation=identity-%s\n' "$$" "$count" +case "$count" in + 1) printf 'stale: w1:p2 (paused awaiting upstream release)\n' ;; + 2) sleep 0.1; printf 'stale: w1:p2 (endpoint missing or unreadable)\n' ;; + *) trap 'exit 0' TERM INT; while [ ! -e "$FM_STOP_FILE" ]; do sleep 0.02; done ;; +esac +SH + chmod +x "$repo/bin/fm-watch-arm.sh" + out=$(PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" FM_ARM_LOG="$log" FM_STOP_FILE="$stop" node --input-type=module 2>&1 <<'EOF' +import { writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +let tool = null; +const deliveries = []; +const pi = { + on() {}, registerCommand() {}, + registerTool(candidate) { if (candidate.name === "fm_watch_arm_pi") tool = candidate; }, + sendUserMessage: async (content) => { deliveries.push(content); }, +}; +writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); +const mod = await import(pathToFileURL(process.env.PLUGIN).href); +mod.default(pi); +await tool.execute("identity", {}, undefined, undefined, {}); +for (let i = 0; i < 400 && deliveries.length === 0; i += 1) await new Promise((resolve) => setTimeout(resolve, 10)); +if (deliveries.length !== 1) throw new Error(`expected one batched follow-up, got ${deliveries.length}`); +const message = deliveries[0]; +if (!message.includes("paused awaiting upstream release")) { + throw new Error(`the first reason for the endpoint was lost: ${message}`); +} +if (!message.includes("endpoint missing or unreadable")) { + throw new Error(`a differently-typed reason for the same endpoint was collapsed away: ${message}`); +} +writeFileSync(process.env.FM_STOP_FILE, "stop\n"); +EOF + ) + status=$? + expect_code 0 "$status" "Pi batch must keep both reasons for one endpoint" + [ -z "$out" ] || fail "Pi endpoint-identity batching printed output: $out" + pass "Pi batching dedupes an unchanged repeat but keeps a second, differently-typed reason for one endpoint" +} + +test_pi_dead_watcher_arm_is_retired_when_handshake_fails() { + local repo home plugin log stop out status + repo="$TMP_ROOT/pi-dead-watcher-root"; home="$TMP_ROOT/pi-dead-watcher-home" + log="$TMP_ROOT/pi-dead-watcher.log"; stop="$TMP_ROOT/pi-dead-watcher.stop" + mkdir -p "$repo/bin" "$home/state" "$home/config" + install_pi_watch_extension_fixture "$repo" + plugin="$repo/.pi/extensions/fm-primary-pi-watch.ts" + cat > "$repo/bin/fm-watch-arm.sh" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --handling-delivered ]; then + printf 'refused\n' >> "${FM_ARM_LOG:?}" + exit 1 +fi +printf 'arm\n' >> "${FM_ARM_LOG:?}" +count=$(grep -c '^arm$' "$FM_ARM_LOG") +if [ "$count" -eq 1 ]; then + printf 'watcher: started pid=%s (beacon fresh)\n' "$$" + printf 'signal: synthetic actionable close\n' + exit 0 +fi +# A successor whose watcher process is already gone: the pid below belongs to a +# shell that has exited, so the extension must retire this dangling arm. +dead=$(bash -c 'echo $$') +printf 'watcher: started pid=%s (beacon fresh) recovery-generation=dead-watcher\n' "$dead" +trap 'printf "retired\n" >> "$FM_ARM_LOG"; exit 0' TERM INT +while [ ! -e "$FM_STOP_FILE" ]; do sleep 0.02; done +SH + chmod +x "$repo/bin/fm-watch-arm.sh" + out=$(PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" FM_ARM_LOG="$log" FM_STOP_FILE="$stop" node --input-type=module 2>&1 <<'EOF' +import { readFileSync, writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +let tool = null; +let prompt = ""; +const pi = { + on() {}, registerCommand() {}, + registerTool(candidate) { if (candidate.name === "fm_watch_arm_pi") tool = candidate; }, + sendUserMessage: async (content) => { prompt += content; }, +}; +writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); +const mod = await import(pathToFileURL(process.env.PLUGIN).href); +mod.default(pi); +await tool.execute("dead-watcher", {}, undefined, undefined, {}); +const rows = () => { + try { return readFileSync(process.env.FM_ARM_LOG, "utf8").trim().split("\n"); } catch { return []; } +}; +for (let i = 0; i < 400 && !rows().includes("retired"); i += 1) await new Promise((resolve) => setTimeout(resolve, 20)); +if (!prompt.includes("handling delivery confirmation was rejected")) { + throw new Error(`the failed handshake was swallowed: ${prompt}`); +} +if (!rows().includes("retired")) { + throw new Error(`the dangling successor arm was never retired: ${rows().join(" | ")}`); +} +writeFileSync(process.env.FM_STOP_FILE, "stop\n"); +EOF + ) + status=$? + expect_code 0 "$status" "Pi must retire a successor arm whose watcher is dead" + [ -z "$out" ] || fail "Pi dead-watcher retirement test printed output: $out" + pass "Pi retires the successor arm when its watcher died before confirming handling delivery" +} + +test_pi_urgent_detail_survives_a_full_batch() { + local repo home plugin log stop out status + repo="$TMP_ROOT/pi-urgent-overflow-root"; home="$TMP_ROOT/pi-urgent-overflow-home" + log="$TMP_ROOT/pi-urgent-overflow.log"; stop="$TMP_ROOT/pi-urgent-overflow.stop" + mkdir -p "$repo/bin" "$home/state" "$home/config" + install_pi_watch_extension_fixture "$repo" + plugin="$repo/.pi/extensions/fm-primary-pi-watch.ts" + cat > "$repo/bin/fm-watch-arm.sh" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --handling-delivered ]; then exit 0; fi +printf 'arm\n' >> "${FM_ARM_LOG:?}" +count=$(grep -c '^arm$' "$FM_ARM_LOG") +printf 'watcher: started pid=%s (beacon fresh) recovery-generation=overflow-%s\n' "$$" "$count" +if [ "$count" -le 4 ]; then + [ "$count" -eq 1 ] || sleep 0.05 + printf 'stale: test:fm-routine-%s (paused awaiting upstream)\n' "$count" +elif [ "$count" -eq 5 ]; then + sleep 0.05 + printf 'signal: %s/state/urgent.status\n' "$FM_HOME" +else + trap 'exit 0' TERM INT + while [ ! -e "$FM_STOP_FILE" ]; do sleep 0.02; done +fi +SH + chmod +x "$repo/bin/fm-watch-arm.sh" + printf 'blocked [key=route]: captain decision required\n' > "$home/state/urgent.status" + out=$(PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" FM_ARM_LOG="$log" \ + FM_STOP_FILE="$stop" FM_WAKE_BATCH_SECONDS=30 FM_WAKE_BATCH_LIMIT=2 node --input-type=module 2>&1 <<'EOF' +import { writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +let tool = null; +const deliveries = []; +const pi = { + on() {}, registerCommand() {}, + registerTool(candidate) { if (candidate.name === "fm_watch_arm_pi") tool = candidate; }, + sendUserMessage: async (content) => { deliveries.push(content); }, +}; +writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); +const mod = await import(pathToFileURL(process.env.PLUGIN).href); +mod.default(pi); +await tool.execute("overflow", {}, undefined, undefined, {}); +for (let i = 0; i < 400 && deliveries.length === 0; i += 1) await new Promise((resolve) => setTimeout(resolve, 10)); +if (deliveries.length !== 1) throw new Error(`expected one flushed follow-up, got ${deliveries.length}`); +const message = deliveries[0]; +if (!message.includes("urgent.status")) { + throw new Error(`the urgent wake that triggered the flush was omitted behind routine wakes: ${message}`); +} +if (!message.includes("more omitted")) { + throw new Error(`the batch was not actually over the render limit, so nothing was tested: ${message}`); +} +writeFileSync(process.env.FM_STOP_FILE, "stop\n"); +EOF + ) + status=$? + expect_code 0 "$status" "Pi urgent detail must survive a full routine batch" + [ -z "$out" ] || fail "Pi urgent-overflow test printed output: $out" + pass "Pi renders the flush-triggering urgent wake before routine wakes so the limit never omits it" +} + +test_pi_terminal_stale_wake_takes_the_urgent_bypass() { + local repo home plugin log stop out status + repo="$TMP_ROOT/pi-stale-urgent-root"; home="$TMP_ROOT/pi-stale-urgent-home" + log="$TMP_ROOT/pi-stale-urgent.log"; stop="$TMP_ROOT/pi-stale-urgent.stop" + mkdir -p "$repo/bin" "$home/state" "$home/config" + printf 'failed: build broke on the release job\n' > "$home/state/urgent.status" + install_pi_watch_extension_fixture "$repo" + plugin="$repo/.pi/extensions/fm-primary-pi-watch.ts" + cat > "$repo/bin/fm-watch-arm.sh" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --handling-delivered ]; then exit 0; fi +printf 'arm\n' >> "${FM_ARM_LOG:?}" +count=$(grep -c '^arm$' "$FM_ARM_LOG") +printf 'watcher: started pid=%s (beacon fresh) recovery-generation=stale-%s\n' "$$" "$count" +if [ "$count" -eq 1 ]; then + # The watcher's terminal-stale reason for a failed crew names its status file. + printf 'stale: test:fm-urgent (%s/state/urgent.status)\n' "$FM_HOME" +else + trap 'exit 0' TERM INT + while [ ! -e "$FM_STOP_FILE" ]; do sleep 0.02; done +fi +SH + chmod +x "$repo/bin/fm-watch-arm.sh" + out=$(PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" FM_ARM_LOG="$log" \ + FM_STOP_FILE="$stop" FM_WAKE_BATCH_SECONDS=30 node --input-type=module 2>&1 <<'EOF' +import { writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +let tool = null; +const deliveries = []; +const pi = { + on() {}, registerCommand() {}, + registerTool(candidate) { if (candidate.name === "fm_watch_arm_pi") tool = candidate; }, + sendUserMessage: async (content) => { deliveries.push(content); }, +}; +writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); +const mod = await import(pathToFileURL(process.env.PLUGIN).href); +mod.default(pi); +await tool.execute("stale-urgent", {}, undefined, undefined, {}); +for (let i = 0; i < 50 && deliveries.length === 0; i += 1) await new Promise((resolve) => setTimeout(resolve, 10)); +if (deliveries.length !== 1) { + throw new Error("a failed crew's terminal stale waited out the 30-second routine batch window"); +} +if (!deliveries[0].includes("test:fm-urgent")) throw new Error(`stale wake identity missing: ${deliveries[0]}`); +writeFileSync(process.env.FM_STOP_FILE, "stop\n"); +EOF + ) + status=$? + expect_code 0 "$status" "Pi terminal stale urgent bypass" + [ -z "$out" ] || fail "Pi terminal stale urgent bypass printed output: $out" + pass "a terminal stale naming a failed: status file takes the Pi urgent bypass" +} + +test_pi_batch_flushes_when_its_owning_arm_ends() { + local repo home plugin log stop out status + repo="$TMP_ROOT/pi-batch-arm-end-root"; home="$TMP_ROOT/pi-batch-arm-end-home" + log="$TMP_ROOT/pi-batch-arm-end.log"; stop="$TMP_ROOT/pi-batch-arm-end.stop" + mkdir -p "$repo/bin" "$home/state" "$home/config" + printf 'working: still building the release\n' > "$home/state/routine.status" + install_pi_watch_extension_fixture "$repo" + plugin="$repo/.pi/extensions/fm-primary-pi-watch.ts" + cat > "$repo/bin/fm-watch-arm.sh" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --handling-delivered ]; then + printf 'confirm %s\n' "$2" >> "${FM_ARM_LOG:?}" + exit 0 +fi +printf 'arm\n' >> "${FM_ARM_LOG:?}" +count=$(grep -c '^arm$' "$FM_ARM_LOG") +if [ "$count" -eq 1 ]; then + printf 'watcher: started pid=%s (beacon fresh)\n' "$$" + printf 'signal: %s/state/routine.status\n' "$FM_HOME" + exit 0 +fi +if [ "$count" -eq 2 ]; then + # The successor that the routine wake's recovery names. It settles readiness, + # then ends non-actionably while the batch window is still open. + printf 'watcher: started pid=%s (beacon fresh) recovery-generation=arm-end-generation\n' "$$" + sleep 0.4 + exit 0 +fi +trap 'exit 0' TERM INT +while [ ! -e "$FM_STOP_FILE" ]; do sleep 0.02; done +SH + chmod +x "$repo/bin/fm-watch-arm.sh" + # A 300s window: only the arm-end flush can deliver this batch during the test. + out=$(PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" FM_ARM_LOG="$log" \ + FM_STOP_FILE="$stop" FM_WAKE_BATCH_SECONDS=300 node --input-type=module 2>&1 <<'EOF' +import { readFileSync, writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +let tool = null; +const deliveries = []; +const pi = { + on() {}, registerCommand() {}, + registerTool(candidate) { if (candidate.name === "fm_watch_arm_pi") tool = candidate; }, + sendUserMessage: async (content) => { deliveries.push(content); }, +}; +writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); +const mod = await import(pathToFileURL(process.env.PLUGIN).href); +mod.default(pi); +await tool.execute("arm-end", {}, undefined, undefined, {}); +for (let i = 0; i < 600 && deliveries.length === 0; i += 1) await new Promise((resolve) => setTimeout(resolve, 10)); +if (deliveries.length !== 1) { + throw new Error("the batch outlived its owning arm: nothing was delivered when that arm ended"); +} +if (!deliveries[0].includes("routine.status")) { + throw new Error(`the flushed batch lost its queued wake: ${deliveries[0]}`); +} +// The watcher of an ended arm has already exited, so nothing is left to confirm; +// test_pi_arm_end_flush_does_not_invent_a_watcher_failure owns that rule. +const rows = readFileSync(process.env.FM_ARM_LOG, "utf8").trim().split("\n"); +if (rows.some((row) => row.startsWith("confirm "))) { + throw new Error(`the arm-end flush confirmed against an already-exited watcher: ${rows.join(" | ")}`); +} +writeFileSync(process.env.FM_STOP_FILE, "stop\n"); +EOF + ) + status=$? + expect_code 0 "$status" "Pi batch must flush when its owning arm ends" + [ -z "$out" ] || fail "Pi arm-end flush test printed output: $out" + pass "a wake batch is flushed when its owning arm ends without a successor, not a window later" +} + +test_pi_batch_ownership_rotates_to_each_successor_arm() { + local repo home plugin log stop out status + repo="$TMP_ROOT/pi-batch-rotate-root"; home="$TMP_ROOT/pi-batch-rotate-home" + log="$TMP_ROOT/pi-batch-rotate.log"; stop="$TMP_ROOT/pi-batch-rotate.stop" + mkdir -p "$repo/bin" "$home/state" "$home/config" + printf 'working: still building a\n' > "$home/state/a.status" + printf 'working: still building b\n' > "$home/state/b.status" + install_pi_watch_extension_fixture "$repo" + plugin="$repo/.pi/extensions/fm-primary-pi-watch.ts" + cat > "$repo/bin/fm-watch-arm.sh" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --handling-delivered ]; then + printf 'confirm\n' >> "${FM_ARM_LOG:?}" + exit 0 +fi +printf 'arm\n' >> "${FM_ARM_LOG:?}" +count=$(grep -c '^arm$' "$FM_ARM_LOG") +printf 'watcher: started pid=%s (beacon fresh) recovery-generation=rotate-%s\n' "$$" "$count" +case "$count" in + 1) printf 'signal: %s/state/a.status\n' "$FM_HOME" ;; + 2) sleep 0.2; printf 'signal: %s/state/b.status\n' "$FM_HOME" ;; + 3) sleep 0.2; exit 0 ;; + *) trap 'exit 0' TERM INT; while [ ! -e "$FM_STOP_FILE" ]; do sleep 0.02; done ;; +esac +SH + chmod +x "$repo/bin/fm-watch-arm.sh" + # 300s window: after two actionable rotations the batch holds two wakes, and only + # an arm-end flush that followed ownership to arm 3 can deliver them here. + out=$(PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" FM_ARM_LOG="$log" \ + FM_STOP_FILE="$stop" FM_WAKE_BATCH_SECONDS=300 node --input-type=module 2>&1 <<'EOF' +import { writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +let tool = null; +const deliveries = []; +const pi = { + on() {}, registerCommand() {}, + registerTool(candidate) { if (candidate.name === "fm_watch_arm_pi") tool = candidate; }, + sendUserMessage: async (content) => { deliveries.push(content); }, +}; +writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); +const mod = await import(pathToFileURL(process.env.PLUGIN).href); +mod.default(pi); +await tool.execute("rotate", {}, undefined, undefined, {}); +for (let i = 0; i < 800 && deliveries.length === 0; i += 1) await new Promise((resolve) => setTimeout(resolve, 10)); +if (deliveries.length !== 1) { + throw new Error("a batch that spanned an arm rotation was never flushed when the current arm ended"); +} +const message = deliveries[0]; +if (!message.includes("a.status") || !message.includes("b.status")) { + throw new Error(`the rotated batch lost one of its aggregated wakes: ${message}`); +} +writeFileSync(process.env.FM_STOP_FILE, "stop\n"); +EOF + ) + status=$? + expect_code 0 "$status" "Pi batch ownership must follow each successor arm" + [ -z "$out" ] || fail "Pi batch rotation test printed output: $out" + pass "a batch that spans arm rotations is still flushed when the current arm ends" +} + +test_pi_arm_end_flush_does_not_invent_a_watcher_failure() { + local repo home plugin log stop out status + repo="$TMP_ROOT/pi-arm-end-quiet-root"; home="$TMP_ROOT/pi-arm-end-quiet-home" + log="$TMP_ROOT/pi-arm-end-quiet.log"; stop="$TMP_ROOT/pi-arm-end-quiet.stop" + mkdir -p "$repo/bin" "$home/state" "$home/config" + printf 'working: still building the release\n' > "$home/state/routine.status" + install_pi_watch_extension_fixture "$repo" + plugin="$repo/.pi/extensions/fm-primary-pi-watch.ts" + # bin/fm-watch-arm.sh waits on its watcher, so by the time an arm closes that + # watcher pid is already gone and --handling-delivered's liveness gate rejects. + cat > "$repo/bin/fm-watch-arm.sh" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = --handling-delivered ]; then + printf 'confirm-rejected\n' >> "${FM_ARM_LOG:?}" + exit 1 +fi +printf 'arm\n' >> "${FM_ARM_LOG:?}" +count=$(grep -c '^arm$' "$FM_ARM_LOG") +if [ "$count" -eq 1 ]; then + printf 'watcher: started pid=%s (beacon fresh)\n' "$$" + printf 'signal: %s/state/routine.status\n' "$FM_HOME" + exit 0 +fi +if [ "$count" -eq 2 ]; then + dead=$(bash -c 'echo $$') + printf 'watcher: started pid=%s (beacon fresh) recovery-generation=quiet-generation\n' "$dead" + sleep 0.4 + exit 0 +fi +trap 'exit 0' TERM INT +while [ ! -e "$FM_STOP_FILE" ]; do sleep 0.02; done +SH + chmod +x "$repo/bin/fm-watch-arm.sh" + out=$(PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" FM_ARM_LOG="$log" \ + FM_STOP_FILE="$stop" FM_WAKE_BATCH_SECONDS=300 node --input-type=module 2>&1 <<'EOF' +import { readFileSync, writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +let tool = null; +const deliveries = []; +const pi = { + on() {}, registerCommand() {}, + registerTool(candidate) { if (candidate.name === "fm_watch_arm_pi") tool = candidate; }, + sendUserMessage: async (content) => { deliveries.push(content); }, +}; +writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); +const mod = await import(pathToFileURL(process.env.PLUGIN).href); +mod.default(pi); +await tool.execute("quiet", {}, undefined, undefined, {}); +for (let i = 0; i < 600 && deliveries.length === 0; i += 1) await new Promise((resolve) => setTimeout(resolve, 10)); +if (deliveries.length !== 1) throw new Error("the arm-end flush did not deliver the batch"); +const message = deliveries[0]; +if (!message.includes("routine.status")) throw new Error(`the flushed batch lost its wake: ${message}`); +if (message.includes("handling delivery confirmation was rejected")) { + throw new Error(`the arm-end flush invented a watcher failure: ${message}`); +} +const rows = readFileSync(process.env.FM_ARM_LOG, "utf8").trim().split("\n"); +if (rows.includes("confirm-rejected")) { + throw new Error(`the arm-end flush confirmed against an already-exited watcher: ${rows.join(" | ")}`); +} +writeFileSync(process.env.FM_STOP_FILE, "stop\n"); +EOF + ) + status=$? + expect_code 0 "$status" "Pi arm-end flush must not invent a watcher failure" + [ -z "$out" ] || fail "Pi arm-end quiet test printed output: $out" + pass "an arm-end flush skips the confirmation its ended arm can no longer satisfy" +} + test_pi_handling_delivery_failure_is_typed_once() { local repo home plugin log stop out status repo="$TMP_ROOT/pi-handling-fail-root" @@ -2255,7 +2787,16 @@ test_pi_tool_returns_agent_tool_result test_pi_redundant_tool_call_is_owned_noop test_pi_scheduled_retry_call_is_owned_noop test_pi_actionable_close_starts_single_successor_before_delivery +test_pi_batches_and_dedupes_routine_wakes +test_pi_urgent_status_bypasses_batch_window test_pi_handling_delivery_failure_is_typed_once +test_pi_batch_keeps_both_reasons_for_one_endpoint +test_pi_dead_watcher_arm_is_retired_when_handshake_fails +test_pi_urgent_detail_survives_a_full_batch +test_pi_terminal_stale_wake_takes_the_urgent_bypass +test_pi_batch_flushes_when_its_owning_arm_ends +test_pi_batch_ownership_rotates_to_each_successor_arm +test_pi_arm_end_flush_does_not_invent_a_watcher_failure test_pi_hung_successor_falls_back_to_typed_wake test_pi_unretired_successor_falls_back_without_retry test_pi_late_unretired_close_resumes_supervision diff --git a/tests/fm-pr-check-security.test.sh b/tests/fm-pr-check-security.test.sh index 03c6ce688e3..ebd9364d43f 100755 --- a/tests/fm-pr-check-security.test.sh +++ b/tests/fm-pr-check-security.test.sh @@ -808,6 +808,15 @@ test_concurrent_watcher_sees_only_complete_publication() { while [ "$n" -le 3 ]; do dir=$(make_case "concurrent-$n") write_task_meta "$dir" + # This fixture has metadata but deliberately no pane, and the publication + # under test is delayed on purpose, so the watcher's FIRST cycle reaches the + # pane-stale layer before any check exists. The watcher reports an + # unreadable endpoint exactly once per disappearance and a wake ends the + # cycle, so without spending that one report here the bounded run would + # close on an endpoint wake that has nothing to do with this test's + # publication race. Spending it up front leaves the pane layer inert, which + # is the state every other watcher fixture in this file already relies on. + : > "$dir/home/state/.endpoint-missing-firstmate_fm-task-a" cat > "$dir/fakebin/cp" < "$root/AGENTS.md" + # Take the lock so the compact digest runs its drain rather than skipping it. + FM_FAKE_HARNESS=pi run_pi_session_start "$home" "$root" "$fakebin:$BASE_PATH" --source startup >/dev/null + + append_wake "$home/state" stale w1 'stale: fm-sess:w1' \ + || fail "could not queue a wake for the compact recovery" + compact=$(FM_FAKE_HARNESS=pi run_pi_session_start "$home" "$root" "$fakebin:$BASE_PATH" --compact) + + # The Pi compact route spawns fm-sessionstart-run.sh with stderr discarded + # (.pi/extensions/fm-primary-turnend-guard.ts), so the acknowledgement command + # only reaches the recovering session if the digest carries it on stdout. + assert_contains "$compact" "stale: fm-sess:w1" \ + "compact recovery digest did not present the queued wake" + assert_contains "$compact" "WAKE_ACK_REQUIRED" \ + "compact recovery digest dropped the acknowledgement command onto discarded stderr" + assert_contains "$compact" "--recovery-generation" \ + "compact recovery acknowledgement command lost its recovery generation" + pass "the compact recovery digest carries its wake acknowledgement command on stdout" +} + +test_new_session_digest_reprints_open_decisions_it_never_saw() { + local rec root home fakebin first second + rec=$(new_world new-session-decisions) + IFS='|' read -r root home fakebin < "$root/AGENTS.md" + printf 'needs-decision [key=api-shape]: pick REST or RPC\n' > "$home/state/task1.status" + + # Session one presents the decision in full and records that presentation. + first=$(FM_FAKE_HARNESS=pi run_pi_session_start "$home" "$root" "$fakebin:$BASE_PATH" --source startup) + assert_contains "$first" "task1" "the first session digest omitted the open decision" + assert_contains "$first" "[key=api-shape]" "the first session digest omitted the decision key" + + # Session two is a different process with none of that context. The decision + # set is unchanged, so the collapse would hand it a bare count. + second=$(FM_FAKE_HARNESS=pi run_pi_session_start "$home" "$root" "$fakebin:$BASE_PATH" --source startup) + assert_not_contains "$second" "OPEN DECISIONS: unchanged" \ + "a new session digest collapsed an open decision it had never been shown" + assert_contains "$second" "[key=api-shape]" \ + "a new session digest omitted the open decision's key" + assert_contains "$second" "pick REST or RPC" \ + "a new session digest omitted the open decision's note" + assert_contains "$second" "--resolve-key" \ + "a new session digest omitted the resolve instruction" + pass "a new session's digest re-presents open decisions the previous session had already been shown" +} + +test_compact_recovery_digest_reports_away_and_x_mode_state() { + local rec root home fakebin attended away + rec=$(new_world compact-supervision-state) + IFS='|' read -r root home fakebin < "$root/AGENTS.md" + FM_FAKE_HARNESS=pi run_pi_session_start "$home" "$root" "$fakebin:$BASE_PATH" --source startup >/dev/null + + attended=$(FM_FAKE_HARNESS=pi run_pi_session_start "$home" "$root" "$fakebin:$BASE_PATH" --compact) + assert_contains "$attended" "Away mode: inactive" \ + "compact recovery digest did not report away-mode state" + assert_contains "$attended" "X mode: inactive" \ + "compact recovery digest did not report X-mode state" + + # Away mode and X mode both change who owns supervision and what a wake means, + # so a post-compaction digest must say so rather than reading identically. + date '+%s' > "$home/state/.afk" + printf 'FM_POLL=30\n' > "$home/config/x-mode.env" + away=$(FM_FAKE_HARNESS=pi run_pi_session_start "$home" "$root" "$fakebin:$BASE_PATH" --compact) + assert_contains "$away" "Away mode: active" \ + "compact recovery digest hid active away mode while the daemon owned the watcher" + assert_contains "$away" "load /afk" \ + "compact recovery digest omitted the away-mode handover instruction" + assert_contains "$away" "X mode: active" \ + "compact recovery digest hid an active X mode" + assert_contains "$away" "$home/config/x-mode.env" \ + "compact recovery digest omitted the X-mode cadence source requirement" + pass "the compact recovery digest always carries away-mode and X-mode supervision state" +} + test_agents_baseline_stays_at_true_start_and_reemits_on_every_drifted_pi_compact() { local rec root home fakebin startup compact_equal compact_first compact_second clear_out resume_out reset_out baseline baseline_after expected_hash refresh_line bootstrap_line rec=$(new_world agents-refresh) @@ -2439,6 +2530,9 @@ test_portable_timeout_escalates_term_resistant_process test_runtime_bound_leaves_a_healthy_digest_untouched test_runtime_bound_leaves_harness_ancestry_headroom test_reemit_skips_startup_sweeps_but_keeps_the_wake_drain +test_compact_recovery_digest_carries_the_wake_acknowledgement +test_new_session_digest_reprints_open_decisions_it_never_saw +test_compact_recovery_digest_reports_away_and_x_mode_state test_agents_baseline_stays_at_true_start_and_reemits_on_every_drifted_pi_compact test_read_only_pi_compact_refreshes_against_its_own_session_identity test_codex_unreachable_reset_sources_do_not_claim_instruction_refresh diff --git a/tests/fm-sessionstart-nudge.test.sh b/tests/fm-sessionstart-nudge.test.sh index baa4a684624..b6c831cb9ac 100755 --- a/tests/fm-sessionstart-nudge.test.sh +++ b/tests/fm-sessionstart-nudge.test.sh @@ -103,6 +103,34 @@ test_unmarked_linked_worktree_is_silent() { pass "fm-sessionstart-nudge: an unmarked linked task worktree is silent" } +test_registered_crew_worktree_suppresses_digest() { + local base="$TMP_ROOT/crew-base" root="$TMP_ROOT/crew-worktree" out status=0 + fm_git_worktree "$base" "$root" fm/sessionstart-crew + mkdir -p "$base/state" "$root/bin" "$root/state" + : > "$root/AGENTS.md" + cat > "$base/state/sessionstart-crew.meta" </dev/null - assert_present "$root/state/.session-start-complete" \ - "startup did not publish the completion proof needed by $source" - status=0 - out=$(run_hook "$root" --source "$source" /dev/null + out=$(run_hook "$root" --source clear "$root/state/active.meta" + printf 'working: routine tail that compact must omit\n' > "$root/state/active.status" + printf 'private context that compact must omit\n' > "$root/data/captain.md" + run_hook "$root" --source startup /dev/null + status=0 + out=$(run_hook "$root" --source compact "$out" \ || fail "drain $round over a growing log failed" - grep -F 'task1' "$out" | grep -F '[key=api-shape]' | grep -F 'pick REST or RPC' >/dev/null \ - || fail "the buried decision was dropped on growth round $round" + [ "$(cat "$out")" = 'OPEN DECISIONS: unchanged, 1 open' ] \ + || fail "the buried decision did not retain its compact marker on growth round $round: $(cat "$out")" probe_bytes=$(last_probe_bytes "$probe" "$status") [ "$probe_bytes" = "$increment_bytes" ] \ || fail "round $round read $probe_bytes bytes, expected exactly this round's $increment_bytes-byte increment (cost is not bounded)" @@ -215,8 +215,8 @@ test_read_failure_preserves_state_for_retry() { FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" \ || fail "wake drain did not recover after the injected read failure" - grep -F 'task4' "$out" | grep -F '[key=x]' | grep -F 'something important' >/dev/null \ - || fail "the open decision disappeared when presentation reads recovered: $(command cat "$out")" + [ "$(cat "$out")" = 'OPEN DECISIONS: unchanged, 1 open' ] \ + || fail "the open decision did not retain its compact marker when reads recovered: $(command cat "$out")" pass "a failed presentation read preserves status state for retry" } @@ -260,8 +260,8 @@ SH FM_STATE_OVERRIDE="$state" FM_OPEN_DECISIONS_READ_PROBE="$probe" PATH="$fakebin:$PATH" "$DRAIN" > "$out" \ || fail "wake drain failed instead of refolding after the cursor-cache read failure" - grep -F 'task5' "$out" | grep -F '[key=cache]' | grep -F 'authoritative status' >/dev/null \ - || fail "the cursor-cache read failure hid the recurring open decision: $(command cat "$out")" + [ "$(cat "$out")" = 'OPEN DECISIONS: unchanged, 1 open' ] \ + || fail "the cursor-cache read failure lost the compact open-decision marker: $(command cat "$out")" if grep -F 'UNREAD STATUS' "$out" >/dev/null \ || grep -F 'already handled informational status' "$out" >/dev/null; then fail "the cursor-cache read failure replayed handled informational status as new: $(command cat "$out")" @@ -297,8 +297,8 @@ test_pre_fix_cursor_refolds_corr_tagged_decision() { FM_STATE_OVERRIDE="$state" FM_OPEN_DECISIONS_READ_PROBE="$probe" "$DRAIN" > "$out" \ || fail "drain failed while migrating the pre-fix corr-tag cursor" - grep -F 'task7 [key=loan-installment-cadence-amount] needs-decision: pick the cadence' "$out" >/dev/null \ - || fail "the pre-fix cursor hid the corr-tagged decision after migration: $(cat "$out")" + [ "$(cat "$out")" = 'OPEN DECISIONS: unchanged, 1 open' ] \ + || fail "the pre-fix cursor migration lost the unchanged decision marker: $(cat "$out")" probe_bytes=$(last_probe_bytes "$probe" "$status") [ "$probe_bytes" = "$status_bytes" ] \ || fail "the pre-fix cursor read $probe_bytes bytes instead of refolding all $status_bytes authoritative bytes" diff --git a/tests/fm-wake-drain-open-decisions.test.sh b/tests/fm-wake-drain-open-decisions.test.sh index 4db2c40954d..16230552bce 100755 --- a/tests/fm-wake-drain-open-decisions.test.sh +++ b/tests/fm-wake-drain-open-decisions.test.sh @@ -37,6 +37,105 @@ test_buried_decision_still_surfaces() { pass "a needs-decision buried under later routine/other-key lines still reports as open" } +test_unchanged_open_decisions_use_compact_marker() { + local dir state first second + dir=$(make_case unchanged-marker); state="$dir/state" + first="$dir/first.out"; second="$dir/second.out" + printf 'needs-decision [key=route]: choose A or B\n' > "$state/task-marker.status" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$first" || fail "first decision drain failed" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$second" || fail "second decision drain failed" + grep -F 'task-marker [key=route]' "$first" >/dev/null \ + || fail "first decision presentation omitted the full open decision" + [ "$(cat "$second")" = 'OPEN DECISIONS: unchanged, 1 open' ] \ + || fail "unchanged decision did not collapse to the one-line marker: $(cat "$second")" + printf 'blocked [key=infra]: credentials missing\n' >> "$state/task-marker.status" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$second" || fail "changed decision drain failed" + grep -F 'credentials missing' "$second" >/dev/null \ + || fail "a changed decision set did not restore the full block" + pass "unchanged open decisions collapse to one line and changed sets print in full" +} + +test_compact_recovery_always_prints_the_full_open_decisions_block() { + local dir state first compact + dir=$(make_case compact-recovery-decisions); state="$dir/state" + first="$dir/first.out"; compact="$dir/compact.out" + printf 'needs-decision [key=api-shape]: pick REST or RPC\n' > "$state/task1.status" + # A normal drain presents the block and commits the unchanged-collapse marker. + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$first" || fail "first decision drain failed" + grep -F 'task1 [key=api-shape]' "$first" >/dev/null \ + || fail "first decision presentation omitted the full open decision" + + # Compaction destroys the context that collapse depends on, so the recovery + # drain must re-present the decision in full even though the set is unchanged. + FM_STATE_OVERRIDE="$state" "$DRAIN" --compact > "$compact" || fail "compact recovery drain failed" + grep -F 'OPEN DECISIONS: unchanged' "$compact" >/dev/null \ + && fail "compact recovery collapsed open decisions to a bare count: $(cat "$compact")" + grep -F 'task1' "$compact" | grep -F '[key=api-shape]' | grep -F 'pick REST or RPC' >/dev/null \ + || fail "compact recovery lost the decision's task, key, and note: $(cat "$compact")" + grep -F "close one by answering it: bin/fm-send.sh --resolve-key " "$compact" >/dev/null \ + || fail "compact recovery omitted the resolve instruction: $(cat "$compact")" + pass "compact recovery re-presents the full open-decisions block instead of an unchanged count" +} + +test_session_recovery_always_prints_the_full_open_decisions_block() { + local dir state first collapsed recovered + dir=$(make_case session-recovery-decisions); state="$dir/state" + first="$dir/first.out"; collapsed="$dir/collapsed.out"; recovered="$dir/recovered.out" + printf 'needs-decision [key=api-shape]: pick REST or RPC\n' > "$state/task1.status" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$first" || fail "first decision drain failed" + grep -F 'task1 [key=api-shape]' "$first" >/dev/null \ + || fail "first decision presentation omitted the full open decision" + + # An ordinary mid-turn drain runs in the same context that saw the block, so it + # still collapses. + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$collapsed" || fail "second decision drain failed" + [ "$(cat "$collapsed")" = 'OPEN DECISIONS: unchanged, 1 open' ] \ + || fail "an in-context repeat drain stopped collapsing: $(cat "$collapsed")" + + # A session-start / clear re-emit digest runs precisely because that context is + # gone, so it must re-present the decision in full. + FM_STATE_OVERRIDE="$state" "$DRAIN" --session-recovery > "$recovered" \ + || fail "session-recovery drain failed" + grep -F 'OPEN DECISIONS: unchanged' "$recovered" >/dev/null \ + && fail "session recovery collapsed open decisions to a bare count: $(cat "$recovered")" + grep -F 'task1' "$recovered" | grep -F '[key=api-shape]' | grep -F 'pick REST or RPC' >/dev/null \ + || fail "session recovery lost the decision's task, key, and note: $(cat "$recovered")" + grep -F "close one by answering it: bin/fm-send.sh --resolve-key " "$recovered" >/dev/null \ + || fail "session recovery omitted the resolve instruction: $(cat "$recovered")" + pass "a session-recovery drain re-presents the full open-decisions block while in-context repeats still collapse" +} + +test_machine_consumer_drain_does_not_spend_the_open_decisions_presentation() { + local dir state daemon returning recovered + dir=$(make_case machine-consumer-decisions); state="$dir/state" + daemon="$dir/daemon.out"; returning="$dir/returning.out"; recovered="$dir/recovered.out" + printf 'needs-decision [key=api-shape]: pick REST or RPC\n' > "$state/task1.status" + + # The away-mode daemon drains to consume the TSV wake rows and discards the + # presented text, so it shows the block to nobody. + FM_STATE_OVERRIDE="$state" "$DRAIN" --no-presentation-commit > "$daemon" \ + || fail "machine-consumer drain failed" + grep -F 'task1 [key=api-shape]' "$daemon" >/dev/null \ + || fail "the machine-consumer drain changed the drain's own output shape: $(cat "$daemon")" + + # The captain returns from away mode. Nobody has seen the block yet, so the + # collapse must not have been spent on the daemon's behalf. + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$returning" || fail "returning drain failed" + grep -F 'OPEN DECISIONS: unchanged' "$returning" >/dev/null \ + && fail "a drain nobody read spent the open-decisions presentation: $(cat "$returning")" + grep -F 'task1' "$returning" | grep -F '[key=api-shape]' | grep -F 'pick REST or RPC' >/dev/null \ + || fail "the away-mode handover lost the decision's task, key, and note: $(cat "$returning")" + grep -F "close one by answering it: bin/fm-send.sh --resolve-key " "$returning" >/dev/null \ + || fail "the away-mode handover omitted the resolve instruction: $(cat "$returning")" + + # bin/fm-afk-return.sh drains in recovery mode, which re-presents regardless. + FM_STATE_OVERRIDE="$state" "$DRAIN" --session-recovery > "$recovered" \ + || fail "away-return recovery drain failed" + grep -F '[key=api-shape]' "$recovered" >/dev/null \ + || fail "the away-return recovery drain omitted the open decision: $(cat "$recovered")" + pass "a drain whose presentation nobody reads never spends the open-decisions collapse" +} + test_explicit_resolution_closes_it() { local dir state out dir=$(make_case resolved) @@ -216,6 +315,10 @@ test_over_long_decision_note_is_capped_with_a_marker() { } test_buried_decision_still_surfaces +test_unchanged_open_decisions_use_compact_marker +test_compact_recovery_always_prints_the_full_open_decisions_block +test_session_recovery_always_prints_the_full_open_decisions_block +test_machine_consumer_drain_does_not_spend_the_open_decisions_presentation test_over_long_decision_note_is_capped_with_a_marker test_explicit_resolution_closes_it test_later_unrelated_terminal_line_does_not_close_it diff --git a/tests/fm-wake-drain-unread-status.test.sh b/tests/fm-wake-drain-unread-status.test.sh index e9cc0a780d7..9b1bb83b6b9 100755 --- a/tests/fm-wake-drain-unread-status.test.sh +++ b/tests/fm-wake-drain-unread-status.test.sh @@ -288,6 +288,22 @@ test_empty_queue_does_not_swallow_later_signal_annotation() { pass "an empty-queue drain preserves routine status for a later signal annotation" } +test_compact_drain_preserves_unread_status_for_ordinary_delivery() { + local dir state compact ordinary + dir=$(make_case compact-preserves-unread); state="$dir/state" + compact="$dir/compact.out"; ordinary="$dir/ordinary.out" + prime_cursor "$state" "$state/compact-task.status" + printf 'note: unread routine line survives compaction\n' >> "$state/compact-task.status" + FM_STATE_OVERRIDE="$state" "$DRAIN" --compact > "$compact" \ + || fail "compact drain failed" + [ ! -s "$compact" ] || fail "compact drain printed a routine status tail: $(cat "$compact")" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$ordinary" \ + || fail "ordinary drain after compact failed" + grep -F 'note: unread routine line survives compaction' "$ordinary" >/dev/null \ + || fail "compact drain advanced the unread-status cursor: $(cat "$ordinary")" + pass "compact recovery omits routine status without consuming its exact-once ordinary presentation" +} + # An absorbed-status receipt is bound to one file identity and byte endpoint. # When it can no longer describe the current file - id reuse, an out-of-band # replacement, a restore, or truncation - it must be dropped and treated as @@ -360,16 +376,48 @@ test_routine_working_lines_stay_silent_on_the_empty_queue() { pass "routine working/done lines still print nothing on an empty-queue drain" } +test_machine_consumer_drain_leaves_unread_status_unread() { + local dir state daemon returning + dir=$(make_case machine-consumer-unread); state="$dir/state" + daemon="$dir/daemon.out"; returning="$dir/returning.out" + # An established home: one presenting drain has already run, so the presentation + # cursor exists and this note is the only unread span. + printf 'note: first line\n' > "$state/task1.status" + FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null || fail "priming drain failed" + printf 'note: upstream contract changed, see thread\n' >> "$state/task1.status" + + # The away-mode daemon drains to consume the TSV wake rows and throws the + # presented text away, so nobody was shown this line. + FM_STATE_OVERRIDE="$state" "$DRAIN" --no-presentation-commit > "$daemon" \ + || fail "machine-consumer drain failed" + grep -F 'upstream contract changed' "$daemon" >/dev/null \ + || fail "the machine-consumer drain changed the drain's own output shape: $(cat "$daemon")" + + # The captain returns from away mode. Exact-once means presented once to a + # READER, so the line must still be unread here. + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$returning" || fail "returning drain failed" + grep -F 'upstream contract changed' "$returning" >/dev/null \ + || fail "a drain nobody read spent the unread-status cursor: $(cat "$returning")" + + # A presenting drain does spend it, so exact-once still holds for readers. + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$dir/third.out" || fail "third drain failed" + grep -F 'upstream contract changed' "$dir/third.out" >/dev/null \ + && fail "an already-presented status line replayed to the same reader: $(cat "$dir/third.out")" + pass "a drain whose presentation nobody reads leaves UNREAD STATUS unread" +} + test_incident_note_answer_buried_under_routine_note_surfaces_both test_already_presented_notes_are_not_replayed test_brand_new_note_after_presentation_is_surfaced test_signal_annotation_surfaces_every_unread_note_not_only_the_newest test_pending_reply_resolution_surfaces_once +test_machine_consumer_drain_leaves_unread_status_unread test_unread_output_over_cap_remains_recoverable test_snapshot_does_not_ack_a_later_append test_retired_task_id_starts_new_status_unread test_open_decisions_fold_is_unchanged test_empty_queue_does_not_swallow_later_signal_annotation +test_compact_drain_preserves_unread_status_for_ordinary_delivery test_unverifiable_absorbed_receipt_still_presents_every_section test_out_of_range_absorbed_receipt_still_presents_every_section test_routine_working_lines_stay_silent_on_the_empty_queue diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 60f46443eb4..cbabf3cab40 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -838,6 +838,131 @@ test_secondmate_status_note_surfaced_despite_busy_agent() { pass "a secondmate's status note surfaces even while its own agent is busy" } +test_repeat_presented_pause_signal_is_absorbed_until_it_changes() { + local dir state fakebin out capture_file window key pid + dir=$(make_case repeat-presented-pause); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; window='test:fm-repeat-pause' + key=$(printf '%s' "$window" | tr '.:/' '___') + printf 'idle pause pane\n' > "$capture_file" + # An ordinary crewmate: a kind=secondmate .status is the mate's routed-reply + # channel and is never absorbable, which test_secondmate_presented_pause_status_always_wakes owns. + printf 'window=%s\nkind=ship\nharness=grok\nbackend=tmux\n' "$window" > "$state/repeat-pause.meta" + printf 'paused: awaiting upstream release\n' > "$state/repeat-pause.status" + export FM_FAKE_CREW_STATE='state: unknown · source: none · no current-state source available' + + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW="$window" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" + pid=$! + wait_for_exit "$pid" 100 || fail "the first pause declaration did not surface" + ack_stopped_cycle "$state" || fail "could not acknowledge the first pause declaration" + + printf 'paused: awaiting upstream release\n' >> "$state/repeat-pause.status" + # Drop the pane-hash bookkeeping the surfacing cycle recorded so this round + # exercises the signal path alone, not a separate pane-stale classification. + rm -f "$state/.hash-$key" "$state/.count-$key" + : > "$out" + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW="$window" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" FM_PAUSE_RESURFACE_SECS=999 + pid=$! + wait_poll_cycle "$state" "$pid" \ + || { reap "$pid"; fail "an unchanged presented pause woke again: $(cat "$out")"; } + [ ! -s "$state/.wake-queue" ] || fail "an unchanged presented pause queued another wake" + reap "$pid" + ack_stopped_cycle "$state" || fail "could not acknowledge the intentional repeat-pause stop" + + printf 'failed: upstream wait ended in failure\n' >> "$state/repeat-pause.status" + rm -f "$state/.hash-$key" "$state/.count-$key" + : > "$out" + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW="$window" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" FM_PAUSE_RESURFACE_SECS=999 + pid=$! + wait_for_exit "$pid" 100 || fail "a changed terminal verb behind a pause was absorbed" + grep -F 'signal:' "$out" >/dev/null || fail "changed terminal verb did not wake immediately" + unset FM_FAKE_CREW_STATE + pass "repeat unchanged pause signals absorb after presentation while changed terminal verbs wake immediately" +} + +test_secondmate_presented_pause_status_always_wakes() { + local dir state fakebin out capture_file window pid + dir=$(make_case mate-presented-pause-wakes); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; window='test:fm-mate-reply' + printf 'idle mate pane\n' > "$capture_file" + printf 'window=%s\nkind=secondmate\n' "$window" > "$state/mate-reply.meta" + printf 'captain-held [key=route]: waiting on decision D\n' > "$state/mate-reply.status" + export FM_FAKE_CREW_STATE='state: unknown · source: none · no current-state source available' + + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW="$window" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" + pid=$! + wait_for_exit "$pid" 100 || fail "the mate's first hold declaration did not surface" + ack_stopped_cycle "$state" || fail "could not acknowledge the mate's first hold declaration" + + # A mate mirrors the identical line again. Its .status is the routed-reply + # channel, so an unchanged repeat is still parent-directed content and must + # wake even though the declaration was already presented. + printf 'captain-held [key=route]: waiting on decision D\n' >> "$state/mate-reply.status" + : > "$out" + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW="$window" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" + pid=$! + wait_for_exit "$pid" 100 \ + || { reap "$pid"; fail "a secondmate's routed-reply status was absorbed as an already-presented repeat"; } + grep -F "signal: $state/mate-reply.status" "$out" >/dev/null \ + || fail "the mate's repeated hold line did not surface as its own signal: $(cat "$out")" + unset FM_FAKE_CREW_STATE + pass "a secondmate status line always wakes, even as an unchanged repeat of a presented hold" +} + +test_urgent_terminal_stale_names_its_status_file() { + local dir state fakebin out capture_file window key pane_hash sig pid + dir=$(make_case urgent-terminal-stale); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; window='test:fm-urgent' + printf 'stopped after the failure\n' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/urgent.meta" + printf 'failed: build broke on the release job\n' > "$state/urgent.status" + sig=$(seen_sig "$state/urgent.status"); printf '%s' "$sig" > "$state/.seen-urgent_status" + key=$(printf '%s' "$window" | tr '.:/' '___') + pane_hash=$(hash_text "stopped after the failure") + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: none · no current-state source available' + + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW="$window" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" + pid=$! + wait_for_exit "$pid" 100 || fail "a failed crew's stale pane did not surface" + # A downstream aggregator classifies urgency by resolving the named status + # file, so the reason has to carry it or the failure waits out the batch window. + grep -F "stale: $window ($state/urgent.status)" "$out" >/dev/null \ + || fail "an urgent terminal stale did not name its status file: $(cat "$out")" + unset FM_FAKE_CREW_STATE + pass "a failed: terminal stale names its status file so an aggregator can classify it urgent" +} + +test_routine_terminal_stale_stays_a_bare_window_identity() { + local dir state fakebin out capture_file window key pane_hash sig pid + dir=$(make_case routine-terminal-stale); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; window='test:fm-routine-done' + printf 'finished, awaiting review\n' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/routine-done.meta" + printf 'done: PR https://example.test/pr/9\n' > "$state/routine-done.status" + sig=$(seen_sig "$state/routine-done.status"); printf '%s' "$sig" > "$state/.seen-routine-done_status" + key=$(printf '%s' "$window" | tr '.:/' '___') + pane_hash=$(hash_text "finished, awaiting review") + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: none · no current-state source available' + + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW="$window" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" + pid=$! + wait_for_exit "$pid" 100 || fail "a done: crew's stale pane did not surface" + grep -Fx "stale: $window" "$out" >/dev/null \ + || fail "a routine terminal stale lost its bare window identity: $(cat "$out")" + unset FM_FAKE_CREW_STATE + pass "a done: terminal stale stays a bare window identity and is not escalated to urgent" +} + test_self_announced_close_does_not_rewake_but_next_note_does() { local dir state fakebin out status_file pid rc dir=$(make_case self-close-quiet); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out" @@ -1125,7 +1250,8 @@ test_nonterminal_stale_paused_absorbed_then_resurfaced() { FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! wait_for_exit "$pid" 100 || fail "watcher did not re-surface a declared pause past the threshold" - grep -F "stale: $window" "$out" >/dev/null || fail "re-surface did not print a stale wake" + grep -F "stale: paused fleet recheck (1 due): $window" "$out" >/dev/null \ + || fail "re-surface did not print the fleet-level stale wake" grep -F "awaiting external" "$out" >/dev/null || fail "re-surface was not labeled a paused/awaiting-external recheck" grep -F "possible wedge" "$out" >/dev/null && fail "a declared pause was mislabeled a possible wedge" [ -e "$state/.paused-resurfaced-$key" ] || fail "the paused re-surface throttle marker was not recorded" @@ -1439,7 +1565,8 @@ test_secondmate_paused_resurfaces_in_normal_mode() { FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! wait_for_exit "$pid" 100 || fail "watcher did not re-surface a paused secondmate" - grep -F "stale: $window" "$out" >/dev/null || fail "paused secondmate did not emit a stale recheck" + grep -F "stale: paused fleet recheck (1 due): $window" "$out" >/dev/null \ + || fail "paused secondmate did not emit the fleet stale recheck" grep -F "awaiting external" "$out" >/dev/null || fail "paused secondmate recheck omitted its external-wait reason" grep -F "awaiting the captain" "$out" >/dev/null && fail "paused secondmate recheck named the captain instead of its external dependency" grep -F "possible wedge" "$out" >/dev/null && fail "paused secondmate was mislabeled a wedge" @@ -1473,7 +1600,8 @@ test_secondmate_captain_held_resurfaces_in_normal_mode() { FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! wait_for_exit "$pid" 100 || fail "watcher did not re-surface a captain-held secondmate" - grep -F "stale: $window" "$out" >/dev/null || fail "captain-held secondmate did not emit a stale recheck" + grep -F "stale: paused fleet recheck (1 due): $window" "$out" >/dev/null \ + || fail "captain-held secondmate did not emit the fleet stale recheck" grep -F "awaiting the captain" "$out" >/dev/null || fail "captain-held secondmate recheck did not name the captain as the blocker: $(cat "$out")" grep -F "awaiting external" "$out" >/dev/null && fail "captain-held secondmate recheck claimed an external wait" grep -F "possible wedge" "$out" >/dev/null && fail "captain-held secondmate was mislabeled a wedge" @@ -1481,6 +1609,336 @@ test_secondmate_captain_held_resurfaces_in_normal_mode() { pass "a captain-held secondmate re-surfaces on the bounded normal-mode cadence" } +test_due_declared_waits_batch_into_one_fleet_wake() { + local dir state fakebin out capture_file back task window key pane_hash sig pid wakes + dir=$(make_case declared-wait-fleet-batch); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + printf 'idle declared wait\n' > "$capture_file" + back=$(( $(date +%s) - 500 )) + pane_hash=$(hash_text "idle declared wait") + for task in paused-one paused-two; do + window="test:fm-$task" + printf 'window=%s\nkind=secondmate\n' "$window" > "$state/$task.meta" + printf 'paused: awaiting upstream for %s\n' "$task" > "$state/$task.status" + set_mtime "$back" "$state/$task.status" + sig=$(seen_sig "$state/$task.status") + printf '%s' "$sig" > "$state/.seen-${task}_status" + key=$(printf '%s' "$window" | tr '.:/' '___') + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + done + export FM_FAKE_CREW_STATE='state: paused · source: status-log · awaiting upstream' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW='test:fm-paused-one' FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher did not deliver the fleet pause batch" + grep -F 'stale: paused fleet recheck (2 due):' "$out" >/dev/null \ + || fail "due declared waits were not summarized in one fleet wake: $(cat "$out")" + grep -F 'test:fm-paused-one' "$out" >/dev/null || fail "fleet pause batch omitted the first pane" + grep -F 'test:fm-paused-two' "$out" >/dev/null || fail "fleet pause batch omitted the second pane" + wakes=$(grep -c "$(printf '\tstale\t')" "$state/.wake-queue" 2>/dev/null || true) + [ "$wakes" -eq 1 ] || fail "fleet pause batch queued $wakes records instead of one" + unset FM_FAKE_CREW_STATE + pass "all due declared waits are reconciled per pane and delivered in one fleet stale wake" +} + +test_omitted_declared_wait_keeps_its_bounded_recheck() { + local dir state fakebin out capture_file back task window key pane_hash sig pid stamped + dir=$(make_case declared-wait-omitted-throttle); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + printf 'idle declared wait\n' > "$capture_file" + back=$(( $(date +%s) - 500 )) + pane_hash=$(hash_text "idle declared wait") + for task in paused-one paused-two; do + window="test:fm-$task" + printf 'window=%s\nkind=secondmate\n' "$window" > "$state/$task.meta" + printf 'paused: awaiting upstream for %s\n' "$task" > "$state/$task.status" + set_mtime "$back" "$state/$task.status" + sig=$(seen_sig "$state/$task.status") + printf '%s' "$sig" > "$state/.seen-${task}_status" + key=$(printf '%s' "$window" | tr '.:/' '___') + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + done + export FM_FAKE_CREW_STATE='state: paused · source: status-log · awaiting upstream' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW='test:fm-paused-one' FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_PAUSE_RESURFACE_SECS=240 FM_PAUSED_RESURFACE_BATCH_LIMIT=1 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher did not deliver the limited fleet pause batch" + grep -F 'stale: paused fleet recheck (2 due):' "$out" >/dev/null \ + || fail "the limited batch did not count both due panes: $(cat "$out")" + grep -F '1 more omitted' "$out" >/dev/null \ + || fail "the limited batch did not report the omitted pane: $(cat "$out")" + stamped=0 + for task in paused-one paused-two; do + key=$(printf '%s' "test:fm-$task" | tr '.:/' '___') + [ -e "$state/.paused-resurfaced-$key" ] && stamped=$((stamped + 1)) + done + [ "$stamped" -eq 1 ] \ + || fail "throttle markers were stamped for $stamped panes; only the pane actually named may be throttled" + unset FM_FAKE_CREW_STATE + pass "a pane omitted from the fleet batch keeps its bounded recheck due instead of being throttled" +} + +test_missing_endpoints_batch_into_one_fleet_wake() { + local dir state fakebin out task window sig pid wakes missing + dir=$(make_case missing-endpoint-fleet-batch); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; missing="$dir/absent-pane-capture" + for task in gone-one gone-two; do + window="test:fm-$task" + printf 'window=%s\nkind=secondmate\n' "$window" > "$state/$task.meta" + printf 'paused: waiting safely but endpoint vanished\n' > "$state/$task.status" + sig=$(seen_sig "$state/$task.status") + printf '%s' "$sig" > "$state/.seen-${task}_status" + done + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW='test:fm-gone-one' \ + FM_FAKE_TMUX_CAPTURE="$missing" FM_FAKE_TMUX_CAPTURE_FAIL=1 FM_PAUSE_RESURFACE_SECS=999 + pid=$! + wait_for_exit "$pid" 100 || fail "a fleet of unreadable endpoints did not wake in the same cycle" + grep -F 'stale: fleet endpoints missing or unreadable (2):' "$out" >/dev/null \ + || fail "unreadable endpoints were not collected into one fleet wake: $(cat "$out")" + grep -F 'test:fm-gone-one' "$out" >/dev/null || fail "fleet endpoint wake omitted the first window" + grep -F 'test:fm-gone-two' "$out" >/dev/null || fail "fleet endpoint wake omitted the second window" + wakes=$(grep -c "$(printf '\tstale\t')" "$state/.wake-queue" 2>/dev/null || true) + [ "$wakes" -eq 1 ] || fail "the fleet endpoint wake queued $wakes records instead of one" + pass "every unreadable endpoint in one cycle is named by a single immediate fleet wake" +} + +test_paused_missing_endpoint_wakes_immediately() { + local dir state fakebin out window sig pid missing + dir=$(make_case paused-missing-endpoint); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; window='test:fm-paused-missing'; missing="$dir/absent-pane-capture" + printf 'window=%s\nkind=secondmate\n' "$window" > "$state/paused-missing.meta" + printf 'paused: waiting safely but endpoint vanished\n' > "$state/paused-missing.status" + sig=$(seen_sig "$state/paused-missing.status") + printf '%s' "$sig" > "$state/.seen-paused-missing_status" + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW="$window" \ + FM_FAKE_TMUX_CAPTURE="$missing" FM_FAKE_TMUX_CAPTURE_FAIL=1 FM_PAUSE_RESURFACE_SECS=999 + pid=$! + wait_for_exit "$pid" 100 || fail "a paused pane with an unreadable endpoint did not wake immediately" + grep -F "stale: $window (endpoint missing or unreadable)" "$out" >/dev/null \ + || fail "paused missing-endpoint wake lost its immediate typed reason: $(cat "$out")" + pass "a declared wait never delays missing or unreadable endpoint detection" +} + +test_presented_declared_wait_still_spends_its_liveness_gate() { + local dir state fakebin out capture_file statusf window key pane_hash pid + dir=$(make_case presented-wait-liveness-gate); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/gatewait.status" + window='test:fm-gatewait' + printf 'idle at an interactive permission prompt\n' > "$capture_file" + printf 'window=%s\nkind=ship\nharness=grok\nbackend=tmux\n' "$window" > "$state/gatewait.meta" + printf 'paused: waiting on upstream\n' > "$statusf" + key=$(printf '%s' "$window" | tr '.:/' '___') + pane_hash=$(hash_text "idle at an interactive permission prompt") + + # The declaration arrives as a status signal and is delivered, which records the + # presentation. Delivering a wake spends no live-agent inspection. + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_FAKE_TMUX_CURRENT_COMMAND=grok \ + FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting on upstream' \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "the first declared wait was not delivered" + grep -F "signal: $statusf" "$out" >/dev/null || fail "the declaration did not surface: $(cat "$out")" + ack_stopped_cycle "$state" || fail "could not acknowledge the delivered declaration" + + # The agent is still alive and stalled at an interactive gate. Its first stale + # hash must still cost one live-agent inspection and surface, not be absorbed + # onto the hour-long declared-wait cadence. + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + : > "$out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_FAKE_TMUX_CURRENT_COMMAND=grok \ + FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting on upstream' \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 \ + || { reap "$pid"; fail "a live crew at a decision gate was silenced by the declared-wait cadence a delivered signal armed"; } + grep -Fx "stale: $window" "$out" >/dev/null \ + || fail "the live decision gate did not surface on its first stale hash: $(cat "$out")" + pass "a declared wait presented through a signal still spends its live-agent inspection on the first stale hash" +} + +test_endpoint_batch_keeps_its_own_bound() { + local dir state fakebin out task window sig pid missing + dir=$(make_case endpoint-own-limit); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; missing="$dir/absent-pane-capture" + for task in bound-one bound-two; do + window="test:fm-$task" + printf 'window=%s\nkind=secondmate\n' "$window" > "$state/$task.meta" + printf 'paused: waiting safely but endpoint vanished\n' > "$state/$task.status" + sig=$(seen_sig "$state/$task.status"); printf '%s' "$sig" > "$state/.seen-${task}_status" + done + # Tightening the PAUSED batch must not silently truncate the endpoint wake. + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW='test:fm-bound-one' \ + FM_FAKE_TMUX_CAPTURE="$missing" FM_FAKE_TMUX_CAPTURE_FAIL=1 \ + FM_PAUSE_RESURFACE_SECS=999 FM_PAUSED_RESURFACE_BATCH_LIMIT=1 + pid=$! + wait_for_exit "$pid" 100 || fail "the endpoint fleet wake was not delivered" + grep -F 'test:fm-bound-one' "$out" >/dev/null || fail "endpoint wake omitted the first window: $(cat "$out")" + grep -F 'test:fm-bound-two' "$out" >/dev/null \ + || fail "the paused batch limit silently truncated the endpoint wake: $(cat "$out")" + grep -F 'more omitted' "$out" >/dev/null && fail "the endpoint wake was bounded by the paused limit: $(cat "$out")" + pass "the missing-endpoint fleet wake is bounded by its own limit, not the paused batch limit" +} + +test_afk_declared_wait_hands_off_undecorated_window_identity() { + local dir state fakebin out drain_out capture_file back task window key pane_hash sig pid + dir=$(make_case afk-declared-wait-no-batch); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + printf 'idle declared wait\n' > "$capture_file" + back=$(( $(date +%s) - 500 )) + pane_hash=$(hash_text "idle declared wait") + for task in afk-one afk-two; do + window="test:fm-$task" + printf 'window=%s\nkind=secondmate\n' "$window" > "$state/$task.meta" + printf 'paused: awaiting upstream for %s\n' "$task" > "$state/$task.status" + set_mtime "$back" "$state/$task.status" + sig=$(seen_sig "$state/$task.status"); printf '%s' "$sig" > "$state/.seen-${task}_status" + key=$(printf '%s' "$window" | tr '.:/' '___') + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + done + date '+%s' > "$state/.afk" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW='test:fm-afk-one' FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_FAKE_CREW_STATE='state: paused · source: status-log · awaiting upstream' \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "AFK declared wait did not hand a stale wake to the daemon" + grep -F 'paused fleet recheck' "$out" >/dev/null \ + && fail "AFK watcher emitted a batched fleet reason the daemon cannot parse into a window: $(cat "$out")" + grep -E '^stale: test:fm-afk-(one|two) \(' "$out" >/dev/null \ + || fail "AFK declared wait lost its per-window stale identity: $(cat "$out")" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the AFK declared wait failed" + grep "$(printf '\tstale\t')" "$drain_out" | grep -E 'stale: test:fm-afk-(one|two) \(' >/dev/null \ + || fail "AFK declared wait was not queued with a daemon-parseable window identity" + pass "away mode keeps the undecorated per-window declared-wait identity instead of the attended fleet batch" +} + +test_afk_missing_endpoint_hands_off_undecorated_window_identity() { + local dir state fakebin out task window sig pid missing + dir=$(make_case afk-missing-endpoint); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; missing="$dir/absent-pane-capture" + for task in afk-gone-one afk-gone-two; do + window="test:fm-$task" + printf 'window=%s\nkind=secondmate\n' "$window" > "$state/$task.meta" + printf 'paused: waiting safely but endpoint vanished\n' > "$state/$task.status" + sig=$(seen_sig "$state/$task.status"); printf '%s' "$sig" > "$state/.seen-${task}_status" + done + date '+%s' > "$state/.afk" + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW='test:fm-afk-gone-one' \ + FM_FAKE_TMUX_CAPTURE="$missing" FM_FAKE_TMUX_CAPTURE_FAIL=1 FM_PAUSE_RESURFACE_SECS=999 + pid=$! + wait_for_exit "$pid" 100 || fail "AFK missing endpoint did not wake the daemon" + grep -F 'fleet endpoints missing or unreadable' "$out" >/dev/null \ + && fail "AFK watcher batched endpoint windows into a reason the daemon cannot parse: $(cat "$out")" + grep -E '^stale: test:fm-afk-gone-(one|two) \(endpoint missing or unreadable\)$' "$out" >/dev/null \ + || fail "AFK missing endpoint lost its per-window identity: $(cat "$out")" + pass "away mode keeps the undecorated per-window identity for a missing or unreadable endpoint" +} + +test_transient_endpoint_failure_preserves_the_stale_hash_suppressor() { + local dir state fakebin out capture_file window key pane_hash sig pid rc + dir=$(make_case endpoint-flap-suppressor); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; window='test:fm-flap' + printf 'quiet inconclusive pane\n' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/flap.meta" + printf 'working: mid task\n' > "$state/flap.status" + sig=$(seen_sig "$state/flap.status"); printf '%s' "$sig" > "$state/.seen-flap_status" + key=$(printf '%s' "$window" | tr '.:/' '___') + pane_hash=$(hash_text "quiet inconclusive pane") + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + # This exact hash was already surfaced as an inconclusive stale on an earlier poll. + printf '%s' "$pane_hash" > "$state/.stale-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: none · no current-state source available' + + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW="$window" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" FM_FAKE_TMUX_CAPTURE_FAIL=1 + pid=$! + wait_for_exit "$pid" 100 || fail "a vanished endpoint did not wake" + grep -F "stale: $window (endpoint missing or unreadable)" "$out" >/dev/null \ + || fail "the first disappearance was not reported: $(cat "$out")" + ack_stopped_cycle "$state" || fail "could not acknowledge the first endpoint wake" + + : > "$out" + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW="$window" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" + pid=$! + wait_poll_cycle "$state" "$pid" \ + || { reap "$pid"; fail "the returning endpoint re-woke a stale pane the supervisor already holds: $(cat "$out")"; } + reap "$pid" + ack_stopped_cycle "$state" || fail "could not acknowledge the intentional endpoint-flap stop" + + : > "$out" + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW="$window" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" FM_FAKE_TMUX_CAPTURE_FAIL=1 + pid=$! + wait_for_exit "$pid" 100 || fail "a second disappearance of the same endpoint was never reported" + grep -F "stale: $window (endpoint missing or unreadable)" "$out" >/dev/null \ + || fail "the second disappearance lost its typed reason: $(cat "$out")" + unset FM_FAKE_CREW_STATE + pass "an endpoint flap reports each disappearance once without clobbering the stale-hash suppressor" +} + +test_undelivered_declared_wait_is_not_marked_presented() { + local dir state fakebin out capture_file statusf window key pane_hash sig pid back + dir=$(make_case undelivered-not-presented); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; window='test:fm-undelivered' + statusf="$state/undelivered.status" + printf 'idle declared wait\n' > "$capture_file" + printf 'window=%s\nkind=secondmate\n' "$window" > "$state/undelivered.meta" + printf 'paused: awaiting the original upstream\npaused: renegotiated with a different upstream\n' > "$statusf" + back=$(( $(date +%s) - 500 )) + set_mtime "$back" "$statusf" + sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-undelivered_status" + key=$(printf '%s' "$window" | tr '.:/' '___') + pane_hash=$(hash_text "idle declared wait") + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + # On the bounded cadence with a fresh throttle: this poll absorbs without + # delivering the pane's line, and only the ORIGINAL line was ever presented. + : > "$state/.paused-$key" + date +%s > "$state/.paused-rechecked-$key" + date +%s > "$state/.paused-resurfaced-$key" + printf 'paused: awaiting the original upstream' > "$state/.paused-presented-$key" + export FM_FAKE_CREW_STATE='state: paused · source: status-log · awaiting upstream' + + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW="$window" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" FM_PAUSE_RESURFACE_SECS=240 + pid=$! + wait_poll_cycle "$state" "$pid" \ + || { reap "$pid"; fail "the throttled declared wait was not absorbed: $(cat "$out")"; } + reap "$pid" + ack_stopped_cycle "$state" || fail "could not acknowledge the intentional throttled-pause stop" + + # The crew's changed declaration now arrives as a status signal. It was never + # delivered, so it must not be absorbed as an already-presented repeat. + rm -f "$state/.seen-undelivered_status" + : > "$out" + watch_bg "$state" "$fakebin" "$out" FM_FAKE_TMUX_WINDOW="$window" \ + FM_FAKE_TMUX_CAPTURE="$capture_file" FM_PAUSE_RESURFACE_SECS=240 + pid=$! + wait_for_exit "$pid" 100 \ + || { reap "$pid"; fail "an undelivered changed declaration was absorbed as already presented"; } + grep -F "signal: $statusf" "$out" >/dev/null \ + || fail "the changed declaration did not surface as its own signal: $(cat "$out")" + unset FM_FAKE_CREW_STATE + pass "a declared wait absorbed without delivery is never recorded as presented" +} + test_secondmate_nonpaused_stale_remains_suppressed() { local dir state fakebin out capture_file statusf window key pane_hash sig pid dir=$(make_case secondmate-stale-suppressed); state="$dir/state"; fakebin="$dir/fakebin" @@ -2935,9 +3393,13 @@ test_turn_ended_provably_working_absorbed test_turn_ended_not_working_surfaced test_working_note_not_working_surfaced test_secondmate_status_note_surfaced_despite_busy_agent +test_repeat_presented_pause_signal_is_absorbed_until_it_changes +test_secondmate_presented_pause_status_always_wakes test_self_announced_close_does_not_rewake_but_next_note_does test_actionable_signal_surfaced test_terminal_stale_surfaced +test_urgent_terminal_stale_names_its_status_file +test_routine_terminal_stale_stays_a_bare_window_identity test_stale_terminal_status_overridden_by_active_run test_nonterminal_stale_provably_working_absorbed_then_escalated test_wedge_escalation_marks_demand_deep_inspection_after_threshold @@ -2956,6 +3418,16 @@ test_declared_pause_cadence_survives_pane_churn_and_restarts test_captain_relevant_line_breaks_an_armed_pause_cadence test_secondmate_paused_resurfaces_in_normal_mode test_secondmate_captain_held_resurfaces_in_normal_mode +test_due_declared_waits_batch_into_one_fleet_wake +test_paused_missing_endpoint_wakes_immediately +test_omitted_declared_wait_keeps_its_bounded_recheck +test_missing_endpoints_batch_into_one_fleet_wake +test_presented_declared_wait_still_spends_its_liveness_gate +test_endpoint_batch_keeps_its_own_bound +test_afk_declared_wait_hands_off_undecorated_window_identity +test_afk_missing_endpoint_hands_off_undecorated_window_identity +test_transient_endpoint_failure_preserves_the_stale_hash_suppressor +test_undelivered_declared_wait_is_not_marked_presented test_secondmate_nonpaused_stale_remains_suppressed test_secondmate_unpause_clears_pause_tracking test_nonterminal_stale_pause_transitions_reclassify_unchanged_hash diff --git a/tests/wake-helpers.sh b/tests/wake-helpers.sh index 8e6281a5763..5a25dc1a790 100644 --- a/tests/wake-helpers.sh +++ b/tests/wake-helpers.sh @@ -68,6 +68,7 @@ if [ "${1:-}" = "list-windows" ]; then exit 0 fi if [ "${1:-}" = "capture-pane" ]; then + [ "${FM_FAKE_TMUX_CAPTURE_FAIL:-0}" != 1 ] || exit 1 if [ -n "${FM_FAKE_TMUX_CAPTURE:-}" ]; then cat "$FM_FAKE_TMUX_CAPTURE" fi @@ -161,6 +162,7 @@ case "${1:-}" in [ -n "${FM_FAKE_TMUX_WINDOW:-}" ] && printf '%s\n' "$FM_FAKE_TMUX_WINDOW" exit 0 ;; capture-pane) + [ "${FM_FAKE_TMUX_CAPTURE_FAIL:-0}" != 1 ] || exit 1 # Honor a single-line band capture (-S N -E M, both non-negative) for the # composer reader's non-bordered compatibility fallback; otherwise (e.g. its # structural full-pane scan or fm_pane_is_busy's "-S -40" tail) return the whole capture. -e is accepted and