From 885866956007d6a0de8ec1ec599e20dbfc467030 Mon Sep 17 00:00:00 2001 From: Jair Date: Wed, 29 Jul 2026 11:10:06 -0300 Subject: [PATCH 1/9] feat(quota-watch): pause and resume crew on Claude quota pressure Add bin/fm-quota-watch.sh, an agent-free script meant for cron/launchd (never the schedule/loop skills, which spend a real turn per firing) that reads quota-axi, interrupts every live ship/scout crewmate of this home above a configurable pause threshold, and resumes them once usage drops below a lower hysteresis threshold. Pausing reuses the existing paused: status verb, so no new supervision code was needed for a live session to treat the pane as an expected wait instead of a stale wedge; verified empirically against fm-crew-state.sh both for the idle-paused case and for a resumed/busy pane correctly overriding the stale status line. Also adds the optional bin/fm-quota-watch-install.sh helper (print-only by default) and docs/quota-watch.md. --- .gitignore | 2 + README.md | 1 + bin/fm-quota-watch-install.sh | 112 ++++++++++ bin/fm-quota-watch.sh | 285 ++++++++++++++++++++++++++ bin/fm-test-run.sh | 10 +- docs/configuration.md | 10 + docs/documentation-audiences.json | 5 + docs/quota-watch.md | 113 +++++++++++ docs/scripts.md | 1 + tests/fm-quota-watch.test.sh | 326 ++++++++++++++++++++++++++++++ 10 files changed, 861 insertions(+), 4 deletions(-) create mode 100755 bin/fm-quota-watch-install.sh create mode 100755 bin/fm-quota-watch.sh create mode 100644 docs/quota-watch.md create mode 100755 tests/fm-quota-watch.test.sh diff --git a/.gitignore b/.gitignore index 372af4735f..1f8d76e836 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ config/x-mode.env config/cmux-socket-password config/wedge-alarm config/herdr-presentation-spaces +config/quota-pause-threshold +config/quota-resume-threshold diff --git a/README.md b/README.md index 5a03c4b9b1..9ddcfc42c3 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,7 @@ Firstmate's skills live in two separate places with different audiences: - [docs/configuration.md](docs/configuration.md) - environment variables, `FM_HOME`, runtime backend selection, optional X mode, the files you set, and harness support. - [docs/calm.md](docs/calm.md) - current Pi `/calm` behavior and supported presentation limits. - [docs/wedge-alarm.md](docs/wedge-alarm.md) - configure the active alert for an away-mode escalation delivery that gets stuck. +- [docs/quota-watch.md](docs/quota-watch.md) - install the agent-free cron/launchd quota watcher that pauses and resumes this home's crew. - [docs/tmux-backend.md](docs/tmux-backend.md) - current setup and limits for the tmux reference backend. - [docs/herdr-backend.md](docs/herdr-backend.md) - current setup, safety boundaries, and limits for the experimental Herdr backend. - [docs/zellij-backend.md](docs/zellij-backend.md) - current setup and limits for the experimental Zellij backend. diff --git a/bin/fm-quota-watch-install.sh b/bin/fm-quota-watch-install.sh new file mode 100755 index 0000000000..d2ba91a531 --- /dev/null +++ b/bin/fm-quota-watch-install.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# fm-quota-watch-install.sh - print (default) or install the OS-scheduler entry +# that runs bin/fm-quota-watch.sh periodically. See docs/quota-watch.md. +# +# This is entirely opt-in: merging or pulling firstmate never runs this script, +# and by default it only PRINTS what it would do. Nothing here starts a cron job +# or launchd agent on its own; the captain (or whoever has shell access to this +# machine) reviews the output and, on a separate deliberate step, either pastes +# it into their own crontab, passes --install-crontab, or saves the launchd +# plist and loads it with launchctl. +# +# Usage: +# fm-quota-watch-install.sh print a crontab line (default) +# fm-quota-watch-install.sh --launchd print a launchd plist instead +# fm-quota-watch-install.sh --install-crontab append the crontab line for +# the current user (idempotent: +# refuses if already present) +# fm-quota-watch-install.sh --interval-minutes N cadence (default 5) +# fm-quota-watch-install.sh --help +# +# The printed command resolves `quota-axi` and this repo's own bin/ to absolute +# paths at generation time and bakes them into an explicit PATH, because cron +# and launchd both run with a minimal PATH that will not see a Node version +# manager's install directory. If `quota-axi` moves (a new Node version, a +# reinstall), regenerate the line rather than hand-editing the stale path. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" + +fm_quota_install_usage() { + sed -n '2,26{s/^# \{0,1\}//;p;}' "$SCRIPT_DIR/fm-quota-watch-install.sh" +} + +MODE=crontab-print +INTERVAL=5 +while [ "$#" -gt 0 ]; do + case "$1" in + --launchd) MODE=launchd-print; shift ;; + --install-crontab) MODE=crontab-install; shift ;; + --interval-minutes) + [ "$#" -ge 2 ] || { echo "fm-quota-watch-install.sh: --interval-minutes requires a value" >&2; exit 2; } + INTERVAL=$2 + shift 2 + ;; + --help|-h) fm_quota_install_usage; exit 0 ;; + *) echo "fm-quota-watch-install.sh: unknown argument '$1' (see --help)" >&2; exit 2 ;; + esac +done +case "$INTERVAL" in + ''|*[!0-9]*|0) echo "fm-quota-watch-install.sh: --interval-minutes must be a positive integer, got '$INTERVAL'" >&2; exit 2 ;; +esac + +if ! QUOTA_AXI_BIN=$(command -v quota-axi 2>/dev/null); then + echo "fm-quota-watch-install.sh: quota-axi not found on PATH; install/authenticate it first (quota-axi --allow-keychain-prompt), then rerun" >&2 + exit 1 +fi +QUOTA_AXI_DIR=$(dirname "$QUOTA_AXI_BIN") +WATCH_BIN="$SCRIPT_DIR/fm-quota-watch.sh" +LOG_FILE="$FM_HOME/state/quota-watch.log" +CRON_PATH="/usr/bin:/bin:/usr/sbin:/sbin:$QUOTA_AXI_DIR" +CRON_LINE="*/$INTERVAL * * * * PATH=\"$CRON_PATH\" FM_HOME=\"$FM_HOME\" \"$WATCH_BIN\" >> \"$LOG_FILE\" 2>&1" + +case "$MODE" in + crontab-print) + cat </dev/null | grep -qF "$WATCH_BIN"; then + echo "fm-quota-watch-install.sh: a crontab entry already references $WATCH_BIN; leaving it unchanged" >&2 + exit 0 + fi + { crontab -l 2>/dev/null || true; printf '%s\n' "$CRON_LINE"; } | crontab - + echo "fm-quota-watch-install.sh: installed crontab entry (every $INTERVAL minute(s))" + echo "$CRON_LINE" + ;; + launchd-print) + cat < + + + + Label + io.firstmate.quota-watch + ProgramArguments + + $WATCH_BIN + + EnvironmentVariables + + PATH + $CRON_PATH + FM_HOME + $FM_HOME + + StartInterval + $((INTERVAL * 60)) + StandardOutPath + $LOG_FILE + StandardErrorPath + $LOG_FILE + + +EOF + echo "# Save as ~/Library/LaunchAgents/io.firstmate.quota-watch.plist, then:" >&2 + echo "# launchctl load ~/Library/LaunchAgents/io.firstmate.quota-watch.plist" >&2 + ;; +esac diff --git a/bin/fm-quota-watch.sh b/bin/fm-quota-watch.sh new file mode 100755 index 0000000000..17c6aaf928 --- /dev/null +++ b/bin/fm-quota-watch.sh @@ -0,0 +1,285 @@ +#!/usr/bin/env bash +# fm-quota-watch.sh - lightweight, agent-free Claude quota gate for this +# firstmate home. +# +# Runs from an OS-level scheduler (crontab or launchd), NOT from the `schedule` +# or `loop` skills - both of those dispatch a real model turn per firing, which +# is exactly the cost this script exists to avoid. It never launches an agent +# or a model turn itself: only `quota-axi --json` (a data-only CLI query), file +# reads/writes under this home's own `state/`, and `bin/fm-send.sh` key/text +# delivery to already-running crewmate panes. +# +# What it does, once per invocation: +# 1. Read current claude quota via `quota-axi --provider claude --json` and +# take the MAX percentUsed across all reported windows (session/weekly/ +# credits): if any window is close to exhausted, further turns are +# constrained, so the tightest window governs the pause decision. +# 2. No usable reading (tool missing, auth_required, malformed output, empty +# windows) is treated as "nothing to do" - never a pause, never a crash. +# 3. pct >= pause threshold (default 80, see below): interrupt every LIVE +# kind=ship/kind=scout crewmate of THIS home (never kind=secondmate, which +# has its own lifecycle per AGENTS.md section 6, and never touches +# projects/) with its harness's verified interrupt key +# (.agents/skills/harness-adapters/SKILL.md), send it nothing further, and +# record the pause both durably (state/.quota-paused) and on the crew's own +# status log using the EXISTING `paused: ` verb +# (bin/fm-classify-lib.sh, AGENTS.md section 8) - this is what makes a live +# firstmate/watcher treat the idle pane as an expected wait instead of a +# stale wedge, with no separate supervision code needed: fm-crew-state.sh +# already re-reads the live pane/run-step BEFORE trusting that status line, +# so a crew that resumed and started working again is never stuck reading +# as paused. +# 4. pct < resume threshold (default 65, hysteresis below the pause +# threshold so a reading oscillating near 80 does not flap): send one +# short note to every crewmate THIS script paused (recorded in +# state/.quota-paused) and clear the flag. A crewmate this script did not +# pause (e.g. a pre-existing declared external-wait `paused:`) is left +# alone. +# 5. Otherwise (below pause threshold with no flag, or inside the hysteresis +# band while already paused): no-op. +# +# Idempotent: rerunning while still above the pause threshold does not resend +# interrupts to already-recorded crew and does not duplicate the flag: it only +# picks up crew spawned since the last pause (still-live, not yet recorded). +# Rerunning below the resume threshold with no flag present is a silent no-op. +# +# Configuration (env wins, then local gitignored config/ file, then default): +# FM_QUOTA_PAUSE_THRESHOLD / config/quota-pause-threshold (default 80) +# FM_QUOTA_RESUME_THRESHOLD / config/quota-resume-threshold (default 65) +# Both accept a bare integer 0-100; an invalid value, or a resume threshold not +# strictly below the pause threshold, falls back to the defaults with a +# stderr warning rather than misbehaving quietly. +# +# Test seams (mirrors FM_CREW_STATE_BIN elsewhere in this codebase): +# FM_QUOTA_AXI_BIN quota reader command, default `quota-axi` off PATH. +# FM_QUOTA_SEND_BIN crewmate sender, default the sibling fm-send.sh. +# Standard overrides also apply: FM_HOME, FM_ROOT_OVERRIDE, FM_STATE_OVERRIDE, +# FM_CONFIG_OVERRIDE. +# +# Usage: +# fm-quota-watch.sh run one check-and-act cycle +# fm-quota-watch.sh --status print resolved config and current reading; +# take no action (for verifying cron setup) +# fm-quota-watch.sh --help +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" +CONFIG="${FM_CONFIG_OVERRIDE:-$FM_HOME/config}" + +# shellcheck source=bin/fm-backend.sh +. "$SCRIPT_DIR/fm-backend.sh" + +FM_QUOTA_AXI_BIN="${FM_QUOTA_AXI_BIN:-quota-axi}" +FM_QUOTA_SEND_BIN="${FM_QUOTA_SEND_BIN:-$SCRIPT_DIR/fm-send.sh}" +PAUSE_FLAG="$STATE/.quota-paused" + +fm_quota_watch_usage() { + sed -n '2,58{s/^# \{0,1\}//;p;}' "$SCRIPT_DIR/fm-quota-watch.sh" +} + +STATUS_ONLY=0 +case "${1:-}" in + --help|-h) fm_quota_watch_usage; exit 0 ;; + --status) STATUS_ONLY=1 ;; + '') ;; + *) echo "fm-quota-watch.sh: unknown argument '$1' (see --help)" >&2; exit 2 ;; +esac + +if [ ! -d "$STATE" ]; then + echo "fm-quota-watch.sh: state dir '$STATE' is missing; nothing to watch" >&2 + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "fm-quota-watch.sh: jq is required and was not found on PATH" >&2 + exit 127 +fi + +# --- configuration ----------------------------------------------------------- + +# First non-empty, non-comment line of config/, mirroring the +# config/backend / config/crew-harness reading convention (docs/configuration.md). +fm_quota_watch_config_value() { # + local f="$CONFIG/$1" line + [ -f "$f" ] || return 0 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ''|'#'*) continue ;; + esac + printf '%s' "$line" + return 0 + done < "$f" +} + +# Clamp to a plain integer 0-100, else print the given default. +fm_quota_watch_validate_pct() { # + case "$1" in + ''|*[!0-9]*) printf '%s' "$2"; return ;; + esac + if [ "$1" -gt 100 ]; then + printf '%s' "$2" + else + printf '%s' "$1" + fi +} + +PAUSE_THRESHOLD=${FM_QUOTA_PAUSE_THRESHOLD:-$(fm_quota_watch_config_value quota-pause-threshold)} +[ -n "$PAUSE_THRESHOLD" ] || PAUSE_THRESHOLD=80 +RESUME_THRESHOLD=${FM_QUOTA_RESUME_THRESHOLD:-$(fm_quota_watch_config_value quota-resume-threshold)} +[ -n "$RESUME_THRESHOLD" ] || RESUME_THRESHOLD=65 +PAUSE_THRESHOLD=$(fm_quota_watch_validate_pct "$PAUSE_THRESHOLD" 80) +RESUME_THRESHOLD=$(fm_quota_watch_validate_pct "$RESUME_THRESHOLD" 65) +if [ "$RESUME_THRESHOLD" -ge "$PAUSE_THRESHOLD" ]; then + echo "fm-quota-watch.sh: configured resume threshold ($RESUME_THRESHOLD) must be strictly below the pause threshold ($PAUSE_THRESHOLD); falling back to defaults 65/80" >&2 + PAUSE_THRESHOLD=80 + RESUME_THRESHOLD=65 +fi + +# --- quota reading ------------------------------------------------------------ + +if ! FM_QUOTA_AXI_RESOLVED=$(command -v "$FM_QUOTA_AXI_BIN" 2>/dev/null); then + echo "fm-quota-watch.sh: '$FM_QUOTA_AXI_BIN' not found on PATH; no quota data available, nothing to do (cron/launchd PATH may need extending - see docs)" >&2 + exit 0 +fi + +QUOTA_JSON=$("$FM_QUOTA_AXI_RESOLVED" --provider claude --json 2>/dev/null) +# Max percentUsed across every reported claude window. Empty output means no +# usable window was reported (auth_required, error, or malformed JSON) - the +# jq filter itself never fails loudly, it just yields nothing to act on. +PCT=$(printf '%s' "$QUOTA_JSON" | jq -r ' + ([.providers[]? | select(.provider=="claude") | .windows[]?.percentUsed | numbers] + | if length > 0 then (max | floor | tostring) else empty end) +' 2>/dev/null) + +if [ "$STATUS_ONLY" -eq 1 ]; then + echo "fm-quota-watch.sh: pause_threshold=$PAUSE_THRESHOLD resume_threshold=$RESUME_THRESHOLD" + if [ -n "$PCT" ]; then + echo "fm-quota-watch.sh: current claude usage pct=$PCT" + else + echo "fm-quota-watch.sh: no usable claude quota reading right now (auth_required, missing tool, or malformed output)" + fi + if [ -f "$PAUSE_FLAG" ]; then + echo "fm-quota-watch.sh: fleet is currently quota-paused:" + sed 's/^/ /' "$PAUSE_FLAG" + else + echo "fm-quota-watch.sh: fleet is not quota-paused" + fi + exit 0 +fi + +if [ -z "$PCT" ]; then + echo "fm-quota-watch.sh: no usable claude quota reading (auth_required, missing tool, or malformed output); leaving fleet state unchanged" >&2 + exit 0 +fi + +# --- crew enumeration --------------------------------------------------------- + +# task= lines already recorded in the pause flag, one per line. +fm_quota_watch_flag_tasks() { + [ -f "$PAUSE_FLAG" ] || return 0 + grep '^task=' "$PAUSE_FLAG" 2>/dev/null | cut -d= -f2- +} + +fm_quota_watch_in_list() { # + printf '%s\n' "$2" | grep -qxF "$1" +} + +# Send this harness's verified single interrupt action (harness-adapters +# SKILL.md). Refuses rather than guessing for an unrecognized/empty harness. +fm_quota_watch_interrupt() { # + local id=$1 harness=$2 + case "$harness" in + claude|codex|pi|pi-signed|kimi) + "$FM_QUOTA_SEND_BIN" "$id" --key Escape + ;; + opencode) + "$FM_QUOTA_SEND_BIN" "$id" --key Escape && "$FM_QUOTA_SEND_BIN" "$id" --key Escape + ;; + grok) + "$FM_QUOTA_SEND_BIN" "$id" --key C-c + ;; + *) + echo "fm-quota-watch.sh: unrecognized harness '$harness' for $id; refusing to guess an interrupt key" >&2 + return 1 + ;; + esac +} + +fm_quota_watch_pause() { + local pct=$1 now paused_at prior_paused_at existing_tasks tmp_flag meta id kind harness + now=$(date +%s) + paused_at=$now + if [ -f "$PAUSE_FLAG" ]; then + prior_paused_at=$(fm_meta_get "$PAUSE_FLAG" paused_at) + [ -n "$prior_paused_at" ] && paused_at=$prior_paused_at + fi + existing_tasks=$(fm_quota_watch_flag_tasks) + + tmp_flag=$(mktemp "${TMPDIR:-/tmp}/fm-quota-paused.XXXXXX") || return 1 + { + printf 'paused_at=%s\n' "$paused_at" + printf 'pct=%s\n' "$pct" + } > "$tmp_flag" + + local acted=0 kept=0 + for meta in "$STATE"/*.meta; do + [ -e "$meta" ] || continue + kind=$(fm_meta_get "$meta" kind) + case "$kind" in ship|scout) ;; *) continue ;; esac + id=$(basename "$meta" .meta) + + if fm_quota_watch_in_list "$id" "$existing_tasks"; then + printf 'task=%s\n' "$id" >> "$tmp_flag" + kept=$((kept + 1)) + continue + fi + + harness=$(fm_meta_get "$meta" harness) + if fm_quota_watch_interrupt "$id" "$harness"; then + printf 'paused: quota at %s%%, auto-resume when it clears\n' "$pct" >> "$STATE/$id.status" + printf 'task=%s\n' "$id" >> "$tmp_flag" + acted=$((acted + 1)) + else + echo "fm-quota-watch.sh: warning: could not interrupt $id (harness=${harness:-}); leaving it unmanaged" >&2 + fi + done + + mv "$tmp_flag" "$PAUSE_FLAG" + echo "fm-quota-watch.sh: pct=$pct >= pause threshold $PAUSE_THRESHOLD - paused $acted new crew, $kept already paused" +} + +fm_quota_watch_resume() { + local pct=$1 id meta acted=0 + while IFS= read -r id; do + [ -n "$id" ] || continue + meta="$STATE/$id.meta" + if [ ! -f "$meta" ]; then + echo "fm-quota-watch.sh: $id has no metadata anymore (torn down while paused); skipping resume send" >&2 + continue + fi + if "$FM_QUOTA_SEND_BIN" "$id" "Quota recovered (now ${pct}% used, below the ${RESUME_THRESHOLD}% resume threshold). Continue where you left off."; then + acted=$((acted + 1)) + else + echo "fm-quota-watch.sh: warning: could not deliver the resume message to $id" >&2 + fi + done < <(fm_quota_watch_flag_tasks) + rm -f "$PAUSE_FLAG" + echo "fm-quota-watch.sh: pct=$pct < resume threshold $RESUME_THRESHOLD - resumed $acted crew, cleared the pause" +} + +# --- act ----------------------------------------------------------------- + +if [ "$PCT" -ge "$PAUSE_THRESHOLD" ]; then + fm_quota_watch_pause "$PCT" +elif [ -f "$PAUSE_FLAG" ]; then + if [ "$PCT" -lt "$RESUME_THRESHOLD" ]; then + fm_quota_watch_resume "$PCT" + else + echo "fm-quota-watch.sh: pct=$PCT inside hysteresis band [$RESUME_THRESHOLD,$PAUSE_THRESHOLD) - still paused, waiting to drop below $RESUME_THRESHOLD" + fi +else + echo "fm-quota-watch.sh: pct=$PCT below pause threshold $PAUSE_THRESHOLD - nothing to do" +fi diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 314232ae88..884546233a 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -122,9 +122,10 @@ family_for_basename() { fm-composer-ghost.test.sh|fm-composer-lib.test.sh|\ fm-crew-state.test.sh|fm-decision-hold-lifecycle.test.sh|\ fm-documentation-audiences.test.sh|fm-ensure-agents-md.test.sh|fm-grok-harness.test.sh|\ - fm-kimi-harness.test.sh|fm-herdr-lab.test.sh|fm-lint.test.sh|\ - fm-operational-input.test.sh|fm-pi-primary-types.test.sh|\ - fm-send-popup-settle.test.sh|fm-send-settle.test.sh|\ + fm-kimi-harness.test.sh|fm-herdr-lab.test.sh|fm-instruction-owners.test.sh|fm-lint.test.sh|\ + fm-install-herdr.test.sh|fm-nm-test-contract.test.sh|fm-no-mistakes-ownership.test.sh|\ + fm-operational-input.test.sh|fm-pi-primary-types.test.sh|fm-quota-watch.test.sh|\ + fm-send-popup-settle.test.sh|fm-send-settle.test.sh|fm-stow-contract.test.sh|\ fm-subagent-pretool-check.test.sh|\ fm-supervision-instructions.test.sh|fm-tmux-submit-busy.test.sh|fm-transition-lib.test.sh|\ fm-test-run.test.sh|fm-test-isolation-proof.test.sh) @@ -676,7 +677,8 @@ families_for_changed_path() { bin/fm-decision-hold.sh|bin/fm-supervision*|bin/fm-transition-lib.sh|\ bin/fm-tmux-lib.sh|bin/fm-marker-lib.sh|bin/fm-operational-input.sh|bin/fm-tasks-axi-lib.sh|\ bin/fm-primary-scope-lib.sh|bin/fm-project-mode.sh|bin/fm-promote.sh|\ - bin/fm-ff-lib.sh|bin/fm-gotmp*|bin/*pretool*) + bin/fm-ff-lib.sh|bin/fm-gotmp*|bin/*pretool*|\ + bin/fm-quota-watch.sh|bin/fm-quota-watch-install.sh) printf '%s\n' pure-contract-unit ;; .agents/skills/*/SKILL.md) diff --git a/docs/configuration.md b/docs/configuration.md index 6bf78b7900..90b4689335 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -116,6 +116,12 @@ An absent file means `auto`, i.e. default-on on macOS: the alarm exists precisel A missing or failing channel logs and falls through to the next, never crashing the daemon. See [`wedge-alarm.md`](wedge-alarm.md) for the current channel reference, [`verification/supervision.md`](verification/supervision.md#wedge-alarm-channels) for active evidence, and [`examples/wedge-alarm`](examples/wedge-alarm) for a copyable config. +## Quota watch (config/quota-pause-threshold / config/quota-resume-threshold) + +`bin/fm-quota-watch.sh` is an agent-free script, meant to be run from `cron` or `launchd` rather than the `schedule`/`loop` skills, that pauses this home's live `kind=ship`/`kind=scout` crew when Claude usage crosses a pause threshold (local gitignored `config/quota-pause-threshold` or `FM_QUOTA_PAUSE_THRESHOLD`, default 80) and resumes them once usage drops back below a lower hysteresis resume threshold (`config/quota-resume-threshold` or `FM_QUOTA_RESUME_THRESHOLD`, default 65). +It never touches a `kind=secondmate`. +See [`quota-watch.md`](quota-watch.md) for the full behavior, the install command (`bin/fm-quota-watch-install.sh`), and why no new supervision code was needed to keep a paused pane from being treated as stuck. + ## Gate defaults (.no-mistakes.yaml) The tracked `.no-mistakes.yaml` keeps test evidence outside the repo and pins `commands.lint` to `bin/fm-lint.sh` so local lint matches CI. @@ -462,6 +468,10 @@ FM_MAX_DEFER_SECS=300 # max buffered escalation age before retry pl FM_WEDGE_ALARM_CHANNEL= # override config/wedge-alarm with one active-alert directive for the wedge alarm; off|auto|osascript|herdr|command:; absent = auto (macOS -> an OS notification) FM_WEDGE_ALARM_EXEC= # notifier seam: route every channel (osascript, herdr, command:) through this command as ` `; "discard" fires nothing; unset in production; the daemon defaults it to "discard" when sourced so no test posts a real notification (docs/wedge-alarm.md) FM_WEDGE_ALARM_TIMEOUT_SECS=10 # maximum seconds for each osascript, herdr, override, or command: notifier before its watchdog terminates it and continues to the next channel; invalid or zero values use 10 +FM_QUOTA_PAUSE_THRESHOLD=80 # fm-quota-watch.sh: claude usage pct (0-100) at/above which live ship/scout crew are interrupted and paused +FM_QUOTA_RESUME_THRESHOLD=65 # fm-quota-watch.sh: claude usage pct (0-100) below which paused crew are resumed; must be strictly below FM_QUOTA_PAUSE_THRESHOLD +FM_QUOTA_AXI_BIN=quota-axi # fm-quota-watch.sh: quota reader command, test seam +FM_QUOTA_SEND_BIN= # fm-quota-watch.sh: crewmate sender, test seam; defaults to the sibling bin/fm-send.sh FM_INJECT_FAIL_SLEEP=30 # seconds to back off when the supervisor pane is unavailable FM_INJECT_CONFIRM_RETRIES=3 # daemon Enter-retry attempts after typing a digest once FM_INJECT_CONFIRM_SLEEP=0.5 # seconds between daemon submit checks diff --git a/docs/documentation-audiences.json b/docs/documentation-audiences.json index 54b2190f6c..8350064e9c 100644 --- a/docs/documentation-audiences.json +++ b/docs/documentation-audiences.json @@ -26,6 +26,7 @@ "readmeSetupTargets": [ "docs/configuration.md", "docs/wedge-alarm.md", + "docs/quota-watch.md", "docs/tmux-backend.md", "docs/herdr-backend.md", "docs/zellij-backend.md", @@ -259,6 +260,10 @@ "path": "docs/orca-backend.md", "audience": "operator-current" }, + { + "path": "docs/quota-watch.md", + "audience": "operator-current" + }, { "path": "docs/scripts.md", "audience": "operator-current" diff --git a/docs/quota-watch.md b/docs/quota-watch.md new file mode 100644 index 0000000000..a0b59878f7 --- /dev/null +++ b/docs/quota-watch.md @@ -0,0 +1,113 @@ +# Quota watch + +`bin/fm-quota-watch.sh` pauses this home's live crew when Claude usage runs +high and resumes them once it recovers, without ever spending quota itself to +do the checking. + +## Why a cron/launchd script, not `schedule` or `loop` + +The `schedule` and `loop` skills both dispatch a real cloud/model turn on every +firing, which is exactly the recurring cost this feature exists to avoid. +`fm-quota-watch.sh` is a plain shell script with no model call anywhere in it: +each run is `quota-axi --json` (a data-only CLI query), a few file reads/writes +under this home's own `state/`, and `bin/fm-send.sh` key/text delivery to +already-running crewmate panes. +It is meant to be triggered by the operating system's own scheduler - `cron` or +`launchd` - on a short interval, so a crossing is caught quickly without a +captain or an agent needing to be present. + +## What it does + +Once per invocation: read Claude's current usage, take the highest +`percentUsed` across every reported window (session, weekly, credits - if any +one window is close to exhausted, further turns are constrained), and act: + +- **At or above the pause threshold** (default 80%): interrupt every live + `kind=ship`/`kind=scout` crewmate of this home with its harness's verified + interrupt key, send it nothing further, and record the pause both durably + (`state/.quota-paused`) and on the crew's own status log using the existing + `paused: ` verb (`AGENTS.md` section 8, `bin/fm-classify-lib.sh`). + A `kind=secondmate` is never touched - it has its own lifecycle. +- **Below the resume threshold** (default 65%, deliberately lower than the + pause threshold so a reading oscillating near 80% does not flap crew back + and forth): send one short note to every crewmate this script paused and + clear the flag. A crewmate that declared its own unrelated `paused:` wait is + left alone - only crew recorded in `state/.quota-paused` are resumed. +- **Between the two thresholds, or below the pause threshold with nothing + paused**: no-op. +- **No usable reading** - `quota-axi` missing, `auth_required`, or malformed + output - is always a harmless no-op, never a pause and never a hang. + +Rerunning while still above the pause threshold does not resend interrupts or +duplicate the flag; it only picks up crew spawned since the last pause. + +## Why no separate supervision code was needed + +Pausing works entirely through the vocabulary supervision already understands: +appending `paused: ` to a crew's status log is the same declared +external-wait verb any crew or firstmate itself already uses for a known +bounded wait. +`bin/fm-crew-state.sh` re-reads the live pane/run-step before ever trusting +that status line, so a crew that resumed and started working again reports +`working`, never `paused` - the mechanism cannot go stale once quota recovers +and this script sends the resume note. +No new field was added to `AGENTS.md`. + +## Configuration + +Env wins, then a local gitignored `config/` file (first non-empty, +non-comment line), then the default: + +| Setting | Env | Config file | Default | +|---|---|---|---| +| Pause threshold | `FM_QUOTA_PAUSE_THRESHOLD` | `config/quota-pause-threshold` | 80 | +| Resume threshold | `FM_QUOTA_RESUME_THRESHOLD` | `config/quota-resume-threshold` | 65 | + +Both accept a bare integer 0-100. +An invalid value, or a resume threshold that is not strictly below the pause +threshold, falls back to the defaults with a stderr warning. + +Run `bin/fm-quota-watch.sh --status` at any time to print the resolved +thresholds and current reading without taking any action - useful for +confirming a fresh install before waiting for a real threshold crossing. + +## Installing the schedule (opt-in, one-time, per machine) + +Nothing in this repo installs a cron job or launchd agent on its own - merging +or pulling firstmate changes nothing about your machine's scheduler. +Run `bin/fm-quota-watch-install.sh` yourself once to see the exact line, or add +`--install-crontab` to append it directly (idempotent: it refuses if an entry +for this script is already present). +It resolves `quota-axi` to an absolute path and bakes an explicit `PATH` into +the generated entry, because both `cron` and `launchd` run with a minimal +`PATH` that will not see a Node version manager's install directory on its own. + +```sh +bin/fm-quota-watch-install.sh # print a crontab line +bin/fm-quota-watch-install.sh --install-crontab # append it for the current user +bin/fm-quota-watch-install.sh --launchd # print a launchd plist instead +``` + +A 5-minute default cadence is frequent enough to catch a crossing quickly +relative to the shortest quota window (the multi-hour session window) without +meaningfully adding to system load; `--interval-minutes` overrides it. +If your session backend needs its own CLI on `PATH` (`tmux`, `herdr`, +`zellij`, ...), extend the printed `PATH` value the same way before installing +it, since `fm-quota-watch.sh` calls `bin/fm-send.sh`, which dispatches through +that backend. + +`quota-axi` itself needs Claude auth once per machine: if `--status` (or a +real run) reports no usable reading and `quota-axi --json` shows +`auth_required` for the `claude` provider, run `quota-axi +--allow-keychain-prompt` once yourself and approve Keychain access; the watcher +never attempts this on its own, since an unattended script prompting for +Keychain access on every cron firing would be worse than the problem it +solves. + +## Tests + +`tests/fm-quota-watch.test.sh` covers crossing the pause threshold, idempotent +reruns, picking up crew spawned mid-pause, the hysteresis band, recovery, +`auth_required`/missing-tool/unknown-harness no-ops, and `--status`, all +against fixture quota JSON and a fake sender - no real quota reading, backend, +or live fleet involved. diff --git a/docs/scripts.md b/docs/scripts.md index 6a10d1310a..7ab38bca2b 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -73,6 +73,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-wake-drain.sh` | Atomically drain queued watcher wakes, emit bounded best-effort status-event annotations, then assert watcher liveness | | `fm-wake-lib.sh` | Shared durable wake queue, portable locks, and watcher identity/health helpers | | `fm-classify-lib.sh` | Shared captain-relevant and declared-external-wait wake classification vocabulary | +| `fm-quota-watch.sh` | Agent-free cron/launchd Claude quota gate: pause and resume this home's live ship/scout crew (docs/quota-watch.md) | | `fm-send.sh` | Send one verified literal line or supported key through the target's recorded backend | | `fm-tmux-lib.sh` | Shared tmux pane primitives for busy detection, composer capture, and verified submit | | `fm-peek.sh` | Print a bounded tail of a crewmate endpoint | diff --git a/tests/fm-quota-watch.test.sh b/tests/fm-quota-watch.test.sh new file mode 100755 index 0000000000..e7a7d104ad --- /dev/null +++ b/tests/fm-quota-watch.test.sh @@ -0,0 +1,326 @@ +#!/usr/bin/env bash +# Tests for bin/fm-quota-watch.sh: the agent-free, cron/launchd-driven Claude +# quota gate that pauses and resumes this home's live ship/scout crew. +# +# Every case fakes quota-axi (via FM_QUOTA_AXI_BIN, fed a fixture JSON file +# through FM_TEST_QUOTA_JSON) and fakes the crewmate sender (via +# FM_QUOTA_SEND_BIN, which logs every call instead of touching a real backend), +# so nothing here depends on a real quota reading, a real tmux/herdr pane, or +# this repo's own live fleet. +# +# Matrix: +# (a) crossing the pause threshold interrupts every live ship/scout crew and +# records the pause (flag + status line), never touching a secondmate +# (b) rerunning while still above threshold is idempotent: no resend, no +# duplicate flag entries +# (c) a crew spawned while already paused is picked up on the next high-pct +# run without resending to the ones already recorded +# (d) a reading inside the hysteresis band leaves an existing pause alone +# (e) dropping below the resume threshold sends one note per paused crew and +# clears the flag +# (f) auth_required (empty windows) is a harmless no-op +# (g) a missing quota-axi binary is a harmless no-op +# (h) an unrecognized harness is refused rather than guessed, and is not +# recorded as paused +# (i) --status prints the resolved config and current reading and takes no +# action +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +QUOTA_WATCH="$ROOT/bin/fm-quota-watch.sh" +TMP_ROOT=$(fm_test_tmproot fm-quota-watch-tests) + +# --- fixtures ---------------------------------------------------------------- + +# Minimal quota-axi --provider claude --json shape with one window at . +quota_json_pct() { # + printf '{"providers":[{"provider":"claude","windows":[{"id":"five_hour","percentUsed":%s}],"state":{"status":"fresh"}}]}\n' "$1" +} + +quota_json_auth_required() { + printf '%s\n' '{"providers":[{"provider":"claude","windows":[],"state":{"status":"auth_required","error":"Claude sign-in required"}}]}' +} + +# Build a case sandbox: state/, config/, fakebin/ with a quota-axi stub reading +# FM_TEST_QUOTA_JSON and a fm-send stub that logs to FM_TEST_SEND_LOG. Echoes +# the case dir. +make_case() { + local name=$1 case_dir + case_dir="$TMP_ROOT/$name" + mkdir -p "$case_dir/state" "$case_dir/config" "$case_dir/fakebin" + + cat > "$case_dir/fakebin/quota-axi" <<'SH' +#!/usr/bin/env bash +cat "$FM_TEST_QUOTA_JSON" +SH + chmod +x "$case_dir/fakebin/quota-axi" + + cat > "$case_dir/fakebin/fm-send" <<'SH' +#!/usr/bin/env bash +id=${1:-} +printf '%s\n' "$*" >> "$FM_TEST_SEND_LOG" +[ "${FM_TEST_SEND_FAIL_ID:-}" != "$id" ] +SH + chmod +x "$case_dir/fakebin/fm-send" + + : > "$case_dir/send.log" + printf '%s\n' "$case_dir" +} + +# write_crew_meta [harness] +write_crew_meta() { + local case_dir=$1 id=$2 kind=$3 harness=${4:-claude} + fm_write_meta "$case_dir/state/$id.meta" \ + "window=fm-$id" \ + "worktree=$case_dir/wt-$id" \ + "project=$case_dir/project" \ + "harness=$harness" \ + "kind=$kind" \ + "mode=no-mistakes" \ + "yolo=off" +} + +# run_watch [extra-arg]: run fm-quota-watch.sh against the sandbox. +# Echoes nothing; sets globals RC, OUT, ERR (file paths) for the caller. +run_watch() { + local case_dir=$1 + shift || true + OUT="$case_dir/out.log" + ERR="$case_dir/err.log" + FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$case_dir/state" \ + FM_CONFIG_OVERRIDE="$case_dir/config" \ + FM_QUOTA_AXI_BIN="$case_dir/fakebin/quota-axi" \ + FM_QUOTA_SEND_BIN="$case_dir/fakebin/fm-send" \ + FM_TEST_QUOTA_JSON="$case_dir/quota.json" \ + FM_TEST_SEND_LOG="$case_dir/send.log" \ + FM_QUOTA_PAUSE_THRESHOLD=80 \ + FM_QUOTA_RESUME_THRESHOLD=65 \ + "$QUOTA_WATCH" "$@" > "$OUT" 2> "$ERR" + RC=$? +} + +count_lines() { # + [ -f "$1" ] && wc -l < "$1" | tr -d ' ' || printf '0' +} + +# --- (a) crossing the pause threshold ----------------------------------------- + +test_pause_crosses_threshold() { + local case_dir + case_dir=$(make_case pause-cross) + write_crew_meta "$case_dir" task-a ship claude + write_crew_meta "$case_dir" task-b scout grok + write_crew_meta "$case_dir" sm-1 secondmate claude + quota_json_pct 83 > "$case_dir/quota.json" + + run_watch "$case_dir" + expect_code 0 "$RC" "pause run exits 0" + + assert_present "$case_dir/state/.quota-paused" "flag created on cross" + assert_grep "task=task-a" "$case_dir/state/.quota-paused" "ship crew recorded paused" + assert_grep "task=task-b" "$case_dir/state/.quota-paused" "scout crew recorded paused" + assert_no_grep "task=sm-1" "$case_dir/state/.quota-paused" "secondmate never recorded paused" + + assert_grep "task-a --key Escape" "$case_dir/send.log" "claude crew gets single Escape" + assert_grep "task-b --key C-c" "$case_dir/send.log" "grok crew gets Ctrl-C" + assert_no_grep "sm-1" "$case_dir/send.log" "secondmate never sent anything" + + assert_present "$case_dir/state/task-a.status" "status file written for paused crew" + assert_grep "paused: quota at 83%" "$case_dir/state/task-a.status" "status uses existing paused: verb" + assert_grep "paused: quota at 83%" "$case_dir/state/task-b.status" "status uses existing paused: verb (scout)" + assert_absent "$case_dir/state/sm-1.status" "secondmate gets no status line" + + pass "pause crosses threshold: ship+scout paused, secondmate untouched" +} + +# --- (b) idempotent rerun ----------------------------------------------------- + +test_idempotent_rerun_same_high_pct() { + local case_dir before after + case_dir=$(make_case idempotent) + write_crew_meta "$case_dir" task-a ship claude + quota_json_pct 90 > "$case_dir/quota.json" + + run_watch "$case_dir" + expect_code 0 "$RC" "first pause run exits 0" + before=$(count_lines "$case_dir/send.log") + [ "$before" -eq 1 ] || fail "expected exactly 1 send call after first pause, got $before" + + run_watch "$case_dir" + expect_code 0 "$RC" "second run (still high) exits 0" + after=$(count_lines "$case_dir/send.log") + [ "$after" -eq "$before" ] || fail "rerun at same high pct resent an interrupt (log grew from $before to $after)" + + flag_lines=$(grep -c '^task=task-a$' "$case_dir/state/.quota-paused") + [ "$flag_lines" -eq 1 ] || fail "flag duplicated task-a entry ($flag_lines occurrences)" + + pass "idempotent rerun at same high pct: no resend, no duplicate flag entry" +} + +# --- (c) newly spawned crew picked up while already paused -------------------- + +test_pause_picks_up_newly_spawned_crew() { + local case_dir + case_dir=$(make_case new-crew) + write_crew_meta "$case_dir" task-a ship claude + quota_json_pct 85 > "$case_dir/quota.json" + + run_watch "$case_dir" + expect_code 0 "$RC" "first pause run exits 0" + assert_grep "task=task-a" "$case_dir/state/.quota-paused" "task-a recorded" + + write_crew_meta "$case_dir" task-c ship codex + run_watch "$case_dir" + expect_code 0 "$RC" "second pause run (new crew) exits 0" + + assert_grep "task=task-c" "$case_dir/state/.quota-paused" "newly spawned crew recorded on next pass" + local task_a_calls + task_a_calls=$(grep -c '^task-a ' "$case_dir/send.log") + [ "$task_a_calls" -eq 1 ] || fail "task-a was resent an interrupt when only task-c was new ($task_a_calls calls)" + assert_grep "task-c --key Escape" "$case_dir/send.log" "new codex crew interrupted" + + pass "crew spawned during an active pause is picked up without resending to existing ones" +} + +# --- (d) hysteresis band leaves an existing pause alone ----------------------- + +test_hysteresis_band_no_resume() { + local case_dir + case_dir=$(make_case hysteresis) + write_crew_meta "$case_dir" task-a ship claude + quota_json_pct 85 > "$case_dir/quota.json" + run_watch "$case_dir" + expect_code 0 "$RC" "pause run exits 0" + + quota_json_pct 72 > "$case_dir/quota.json" + run_watch "$case_dir" + expect_code 0 "$RC" "band run exits 0" + + assert_present "$case_dir/state/.quota-paused" "flag remains inside hysteresis band" + assert_contains "$(cat "$case_dir/out.log")" "hysteresis band" "band state is reported" + local send_calls + send_calls=$(count_lines "$case_dir/send.log") + [ "$send_calls" -eq 1 ] || fail "hysteresis band run sent something (expected only the original pause call, got $send_calls total)" + + pass "reading inside the hysteresis band leaves an existing pause untouched" +} + +# --- (e) recovery below resume threshold -------------------------------------- + +test_resume_below_recovery_threshold() { + local case_dir + case_dir=$(make_case resume) + write_crew_meta "$case_dir" task-a ship claude + write_crew_meta "$case_dir" task-b scout opencode + quota_json_pct 88 > "$case_dir/quota.json" + run_watch "$case_dir" + expect_code 0 "$RC" "pause run exits 0" + assert_grep "task-b --key Escape" "$case_dir/send.log" "opencode interrupt uses Escape" + local opencode_escapes + opencode_escapes=$(grep -c '^task-b --key Escape$' "$case_dir/send.log") + [ "$opencode_escapes" -eq 2 ] || fail "opencode interrupt should send Escape twice, got $opencode_escapes" + + quota_json_pct 50 > "$case_dir/quota.json" + run_watch "$case_dir" + expect_code 0 "$RC" "resume run exits 0" + + assert_absent "$case_dir/state/.quota-paused" "flag cleared on recovery" + assert_grep "task-a Quota recovered" "$case_dir/send.log" "task-a gets the resume note" + assert_grep "task-b Quota recovered" "$case_dir/send.log" "task-b gets the resume note" + + pass "dropping below the resume threshold notifies and clears every paused crew" +} + +# --- (f) auth_required is a harmless no-op ------------------------------------ + +test_auth_required_is_harmless_noop() { + local case_dir + case_dir=$(make_case auth-required) + write_crew_meta "$case_dir" task-a ship claude + quota_json_auth_required > "$case_dir/quota.json" + + run_watch "$case_dir" + expect_code 0 "$RC" "auth_required run exits 0, never hangs or crashes" + + assert_absent "$case_dir/state/.quota-paused" "no pause recorded from auth_required" + [ ! -s "$case_dir/send.log" ] || fail "auth_required must never send anything to crew" + assert_absent "$case_dir/state/task-a.status" "no status line written from auth_required" + + pass "auth_required (empty windows) is a harmless no-op" +} + +# --- (g) missing quota-axi binary is a harmless no-op ------------------------- + +test_missing_quota_axi_tool_is_harmless_noop() { + local case_dir + case_dir=$(make_case missing-tool) + write_crew_meta "$case_dir" task-a ship claude + quota_json_pct 90 > "$case_dir/quota.json" + + FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$case_dir/state" \ + FM_CONFIG_OVERRIDE="$case_dir/config" \ + FM_QUOTA_AXI_BIN="$case_dir/fakebin/does-not-exist-quota-axi" \ + FM_QUOTA_SEND_BIN="$case_dir/fakebin/fm-send" \ + FM_TEST_QUOTA_JSON="$case_dir/quota.json" \ + FM_TEST_SEND_LOG="$case_dir/send.log" \ + "$QUOTA_WATCH" > "$case_dir/out.log" 2> "$case_dir/err.log" + RC=$? + + expect_code 0 "$RC" "missing quota-axi binary exits 0, never hangs or crashes" + assert_absent "$case_dir/state/.quota-paused" "no pause recorded when quota-axi is missing" + [ ! -s "$case_dir/send.log" ] || fail "missing quota-axi must never send anything to crew" + + pass "a missing quota-axi binary is a harmless no-op" +} + +# --- (h) unrecognized harness is refused, not guessed ------------------------- + +test_unknown_harness_refuses_to_guess() { + local case_dir + case_dir=$(make_case unknown-harness) + write_crew_meta "$case_dir" task-a ship some-future-harness + quota_json_pct 90 > "$case_dir/quota.json" + + run_watch "$case_dir" + expect_code 0 "$RC" "run with an unknown harness still exits 0" + + [ ! -s "$case_dir/send.log" ] || fail "unknown harness must never be sent a guessed key" + assert_no_grep "task=task-a" "$case_dir/state/.quota-paused" "unrecognized-harness crew is not recorded paused" + assert_contains "$(cat "$case_dir/err.log")" "refusing to guess" "refusal is logged" + + pass "an unrecognized harness is refused rather than guessed, and left unmanaged" +} + +# --- (i) --status is read-only ------------------------------------------------ + +test_status_flag_smoke() { + local case_dir + case_dir=$(make_case status-smoke) + write_crew_meta "$case_dir" task-a ship claude + quota_json_pct 42 > "$case_dir/quota.json" + + run_watch "$case_dir" --status + expect_code 0 "$RC" "--status exits 0" + assert_contains "$(cat "$case_dir/out.log")" "pause_threshold=80" "status prints resolved pause threshold" + assert_contains "$(cat "$case_dir/out.log")" "pct=42" "status prints current reading" + assert_absent "$case_dir/state/.quota-paused" "--status takes no action" + [ ! -s "$case_dir/send.log" ] || fail "--status must never send anything" + + pass "--status reports config/reading without acting" +} + +# --- run ----------------------------------------------------------------- + +test_pause_crosses_threshold +test_idempotent_rerun_same_high_pct +test_pause_picks_up_newly_spawned_crew +test_hysteresis_band_no_resume +test_resume_below_recovery_threshold +test_auth_required_is_harmless_noop +test_missing_quota_axi_tool_is_harmless_noop +test_unknown_harness_refuses_to_guess +test_status_flag_smoke From e31ce24153a57a447f470965069732b10d15aec4 Mon Sep 17 00:00:00 2001 From: Jair Date: Wed, 29 Jul 2026 11:16:51 -0300 Subject: [PATCH 2/9] no-mistakes(review): Export FM_HOME/overrides so fm-send.sh subprocess resolves them --- bin/fm-quota-watch.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/fm-quota-watch.sh b/bin/fm-quota-watch.sh index 17c6aaf928..f6e4fadc43 100755 --- a/bin/fm-quota-watch.sh +++ b/bin/fm-quota-watch.sh @@ -68,6 +68,10 @@ FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" CONFIG="${FM_CONFIG_OVERRIDE:-$FM_HOME/config}" +export FM_HOME +[ -n "${FM_ROOT_OVERRIDE:-}" ] && export FM_ROOT_OVERRIDE +[ -n "${FM_STATE_OVERRIDE:-}" ] && export FM_STATE_OVERRIDE +[ -n "${FM_CONFIG_OVERRIDE:-}" ] && export FM_CONFIG_OVERRIDE # shellcheck source=bin/fm-backend.sh . "$SCRIPT_DIR/fm-backend.sh" From 380951da90bae785487ee98df6f567c132e6631f Mon Sep 17 00:00:00 2001 From: Jair Date: Wed, 29 Jul 2026 11:20:33 -0300 Subject: [PATCH 3/9] no-mistakes(document): docs: add missing fm-quota-watch-install.sh row to scripts.md --- docs/scripts.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/scripts.md b/docs/scripts.md index 7ab38bca2b..da0eeba4f0 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -74,6 +74,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-wake-lib.sh` | Shared durable wake queue, portable locks, and watcher identity/health helpers | | `fm-classify-lib.sh` | Shared captain-relevant and declared-external-wait wake classification vocabulary | | `fm-quota-watch.sh` | Agent-free cron/launchd Claude quota gate: pause and resume this home's live ship/scout crew (docs/quota-watch.md) | +| `fm-quota-watch-install.sh` | Print (default) or, with `--install-crontab`, append the OS-scheduler entry that runs `fm-quota-watch.sh` periodically | | `fm-send.sh` | Send one verified literal line or supported key through the target's recorded backend | | `fm-tmux-lib.sh` | Shared tmux pane primitives for busy detection, composer capture, and verified submit | | `fm-peek.sh` | Print a bounded tail of a crewmate endpoint | From 0711b775f448fb5341cfead46604c23dceb779fb Mon Sep 17 00:00:00 2001 From: Jair Date: Wed, 29 Jul 2026 13:52:20 -0300 Subject: [PATCH 4/9] fix(quota-watch): only session/weekly windows drive the pause decision Captain caught this testing the live script: taking the MAX across every reported window included the credits (paid overage) window, which sat at 79% while the actual session/weekly rate-limit windows were only at 35-51%. That drove premature pauses purely from money spent, working against the actual goal of using the free session/weekly allowance as fully as possible. Restrict the jq filter to windows with kind session or weekly; credits is now excluded entirely, and an absent session/weekly reading still falls back to the existing harmless no-op rather than credits. Updates docs and adds a test covering a high credits reading alongside low session/weekly for both the pause and resume decisions. --- bin/fm-quota-watch.sh | 26 ++++++++++++------ docs/quota-watch.md | 26 ++++++++++++++---- tests/fm-quota-watch.test.sh | 53 ++++++++++++++++++++++++++++++++++-- 3 files changed, 90 insertions(+), 15 deletions(-) diff --git a/bin/fm-quota-watch.sh b/bin/fm-quota-watch.sh index f6e4fadc43..56a8cf813a 100755 --- a/bin/fm-quota-watch.sh +++ b/bin/fm-quota-watch.sh @@ -11,9 +11,12 @@ # # What it does, once per invocation: # 1. Read current claude quota via `quota-axi --provider claude --json` and -# take the MAX percentUsed across all reported windows (session/weekly/ -# credits): if any window is close to exhausted, further turns are -# constrained, so the tightest window governs the pause decision. +# take the MAX percentUsed across the rate-limit-style windows only +# (kind session and weekly - the free allowance this script conserves). +# The credits window (kind "credits") measures PAID overage spend, not +# that free allowance, and is deliberately excluded: pausing crew because +# paid credits are being spent would work against maximizing use of the +# free session/weekly quota, the opposite of the intent. # 2. No usable reading (tool missing, auth_required, malformed output, empty # windows) is treated as "nothing to do" - never a pause, never a crash. # 3. pct >= pause threshold (default 80, see below): interrupt every LIVE @@ -81,7 +84,7 @@ FM_QUOTA_SEND_BIN="${FM_QUOTA_SEND_BIN:-$SCRIPT_DIR/fm-send.sh}" PAUSE_FLAG="$STATE/.quota-paused" fm_quota_watch_usage() { - sed -n '2,58{s/^# \{0,1\}//;p;}' "$SCRIPT_DIR/fm-quota-watch.sh" + sed -n '2,66{s/^# \{0,1\}//;p;}' "$SCRIPT_DIR/fm-quota-watch.sh" } STATUS_ONLY=0 @@ -150,11 +153,18 @@ if ! FM_QUOTA_AXI_RESOLVED=$(command -v "$FM_QUOTA_AXI_BIN" 2>/dev/null); then fi QUOTA_JSON=$("$FM_QUOTA_AXI_RESOLVED" --provider claude --json 2>/dev/null) -# Max percentUsed across every reported claude window. Empty output means no -# usable window was reported (auth_required, error, or malformed JSON) - the -# jq filter itself never fails loudly, it just yields nothing to act on. +# Max percentUsed across only the rate-limit-style windows (kind session or +# weekly - the actual quota this script conserves). The credits window (kind +# "credits", e.g. id extra_usage) measures PAID overage spend, not the free +# session/weekly allowance, and deliberately never drives the pause decision: +# pausing crew because paid credits are being spent would work against +# maximizing use of the free session/weekly quota, the opposite of the intent. +# Empty output means no session/weekly window was reported (auth_required, +# error, malformed JSON, or a schema with neither kind present) - the jq +# filter itself never fails loudly, it just yields nothing to act on. PCT=$(printf '%s' "$QUOTA_JSON" | jq -r ' - ([.providers[]? | select(.provider=="claude") | .windows[]?.percentUsed | numbers] + ([.providers[]? | select(.provider=="claude") | .windows[]? + | select(.kind=="session" or .kind=="weekly") | .percentUsed | numbers] | if length > 0 then (max | floor | tostring) else empty end) ' 2>/dev/null) diff --git a/docs/quota-watch.md b/docs/quota-watch.md index a0b59878f7..95af41d4a8 100644 --- a/docs/quota-watch.md +++ b/docs/quota-watch.md @@ -19,8 +19,8 @@ captain or an agent needing to be present. ## What it does Once per invocation: read Claude's current usage, take the highest -`percentUsed` across every reported window (session, weekly, credits - if any -one window is close to exhausted, further turns are constrained), and act: +`percentUsed` across the rate-limit-style windows only (`kind` `session` and +`weekly`), and act: - **At or above the pause threshold** (default 80%): interrupt every live `kind=ship`/`kind=scout` crewmate of this home with its harness's verified @@ -41,6 +41,21 @@ one window is close to exhausted, further turns are constrained), and act: Rerunning while still above the pause threshold does not resend interrupts or duplicate the flag; it only picks up crew spawned since the last pause. +## Why the credits window is ignored + +`quota-axi`'s claude reading reports three window kinds: `session` (the +multi-hour rate limit), `weekly`, and `credits` (paid overage spend, tracked +separately from the free session/weekly allowance). +Only `session` and `weekly` drive the pause/resume decision. +The `credits` window measures money spent, not the free allowance this script +exists to conserve - pausing crew because paid credits are high would work +against the actual goal of using the free session/weekly quota as fully as +possible, so a high `credits` reading alone never pauses anything and a still-high +`credits` reading never blocks a resume once `session`/`weekly` recover. +If neither `session` nor `weekly` is present in a reading, that is treated the +same as no usable reading at all (see below) rather than falling back to +`credits`. + ## Why no separate supervision code was needed Pausing works entirely through the vocabulary supervision already understands: @@ -108,6 +123,7 @@ solves. `tests/fm-quota-watch.test.sh` covers crossing the pause threshold, idempotent reruns, picking up crew spawned mid-pause, the hysteresis band, recovery, -`auth_required`/missing-tool/unknown-harness no-ops, and `--status`, all -against fixture quota JSON and a fake sender - no real quota reading, backend, -or live fleet involved. +`auth_required`/missing-tool/unknown-harness no-ops, `--status`, and a high +`credits` reading never driving a pause or blocking a resume while +`session`/`weekly` stay low, all against fixture quota JSON and a fake sender - +no real quota reading, backend, or live fleet involved. diff --git a/tests/fm-quota-watch.test.sh b/tests/fm-quota-watch.test.sh index e7a7d104ad..ea94d203c7 100755 --- a/tests/fm-quota-watch.test.sh +++ b/tests/fm-quota-watch.test.sh @@ -24,6 +24,8 @@ # recorded as paused # (i) --status prints the resolved config and current reading and takes no # action +# (j) a high credits (paid overage) window never drives a pause when +# session/weekly are both low - only the rate-limit-style windows count set -u # shellcheck source=tests/lib.sh @@ -34,9 +36,16 @@ TMP_ROOT=$(fm_test_tmproot fm-quota-watch-tests) # --- fixtures ---------------------------------------------------------------- -# Minimal quota-axi --provider claude --json shape with one window at . +# Minimal quota-axi --provider claude --json shape with one kind=session +# window at , matching the real schema's rate-limit-style window shape. quota_json_pct() { # - printf '{"providers":[{"provider":"claude","windows":[{"id":"five_hour","percentUsed":%s}],"state":{"status":"fresh"}}]}\n' "$1" + printf '{"providers":[{"provider":"claude","windows":[{"id":"five_hour","kind":"session","percentUsed":%s}],"state":{"status":"fresh"}}]}\n' "$1" +} + +# All three real window kinds present: session, weekly, and credits (paid +# overage spend). Lets a test independently control each. +quota_json_windows() { # + printf '{"providers":[{"provider":"claude","windows":[{"id":"five_hour","kind":"session","percentUsed":%s},{"id":"seven_day","kind":"weekly","percentUsed":%s},{"id":"extra_usage","kind":"credits","percentUsed":%s}],"state":{"status":"fresh"}}]}\n' "$1" "$2" "$3" } quota_json_auth_required() { @@ -313,6 +322,44 @@ test_status_flag_smoke() { pass "--status reports config/reading without acting" } +# --- (j) the credits window never drives pause/resume ------------------------- + +test_credits_window_never_drives_pause() { + local case_dir + case_dir=$(make_case credits-ignored) + write_crew_meta "$case_dir" task-a ship claude + quota_json_windows 20 30 95 > "$case_dir/quota.json" + + run_watch "$case_dir" --status + expect_code 0 "$RC" "--status exits 0" + assert_contains "$(cat "$case_dir/out.log")" "pct=30" "status reflects max(session,weekly), ignoring the higher credits reading" + + run_watch "$case_dir" + expect_code 0 "$RC" "run with high credits but low session/weekly exits 0" + assert_absent "$case_dir/state/.quota-paused" "a high credits window alone never triggers a pause" + [ ! -s "$case_dir/send.log" ] || fail "a high credits window alone must never send anything to crew" + + pass "a high credits (paid overage) window never drives a pause when session/weekly are low" +} + +test_credits_window_ignored_during_resume_decision() { + local case_dir + case_dir=$(make_case credits-ignored-resume) + write_crew_meta "$case_dir" task-a ship claude + quota_json_windows 85 30 20 > "$case_dir/quota.json" + run_watch "$case_dir" + expect_code 0 "$RC" "pause run (high session) exits 0" + assert_present "$case_dir/state/.quota-paused" "high session window paused the crew" + + quota_json_windows 40 30 99 > "$case_dir/quota.json" + run_watch "$case_dir" + expect_code 0 "$RC" "resume run exits 0" + assert_absent "$case_dir/state/.quota-paused" "session/weekly dropping below the resume threshold resumes crew even while credits is high" + assert_grep "task-a Quota recovered" "$case_dir/send.log" "resume note sent based on session/weekly, not the still-high credits window" + + pass "recovery is decided from session/weekly alone, even while the credits window stays high" +} + # --- run ----------------------------------------------------------------- test_pause_crosses_threshold @@ -324,3 +371,5 @@ test_auth_required_is_harmless_noop test_missing_quota_axi_tool_is_harmless_noop test_unknown_harness_refuses_to_guess test_status_flag_smoke +test_credits_window_never_drives_pause +test_credits_window_ignored_during_resume_decision From 5568a25f1d5cbffaaad391386a3a4ac58f32a6e0 Mon Sep 17 00:00:00 2001 From: Jair Date: Wed, 29 Jul 2026 17:27:19 -0300 Subject: [PATCH 5/9] fix(quota-watch): resolve herdr/tmux/zellij/cmux into the generated PATH Captain found this live in production: the cron job read quota fine (quota-axi resolved) but every interrupt/resume send silently failed with "herdr server ... did not report running" because herdr lives in /opt/homebrew/bin, which the generated crontab PATH never included - fm-quota-watch-install.sh only ever resolved quota-axi's own directory. fm-quota-watch-install.sh now also resolves jq (required, like quota-axi) and every session-backend CLI it finds installed (tmux, herdr, zellij, cmux - included only if actually present) into the generated PATH for both the crontab line and the launchd plist, then re-verifies every resolved binary against that generated PATH under a scrubbed environment before printing anything, so a stale/incomplete PATH is caught at generation time instead of failing silently on every cron firing. Adds tests/fm-quota-watch-install.test.sh, which proves the generated PATH actually resolves each included binary under env -i (the same class of check that would have caught this bug), and updates docs/quota-watch.md. --- bin/fm-quota-watch-install.sh | 79 ++++++++++-- bin/fm-test-run.sh | 1 + docs/quota-watch.md | 32 +++-- tests/fm-quota-watch-install.test.sh | 177 +++++++++++++++++++++++++++ 4 files changed, 274 insertions(+), 15 deletions(-) create mode 100755 tests/fm-quota-watch-install.test.sh diff --git a/bin/fm-quota-watch-install.sh b/bin/fm-quota-watch-install.sh index d2ba91a531..289cb13669 100755 --- a/bin/fm-quota-watch-install.sh +++ b/bin/fm-quota-watch-install.sh @@ -18,11 +18,23 @@ # fm-quota-watch-install.sh --interval-minutes N cadence (default 5) # fm-quota-watch-install.sh --help # -# The printed command resolves `quota-axi` and this repo's own bin/ to absolute -# paths at generation time and bakes them into an explicit PATH, because cron -# and launchd both run with a minimal PATH that will not see a Node version -# manager's install directory. If `quota-axi` moves (a new Node version, a -# reinstall), regenerate the line rather than hand-editing the stale path. +# The printed command resolves `quota-axi`, `jq`, and every session-backend CLI +# fm-send.sh might dispatch through (tmux, herdr, zellij, cmux - whichever are +# actually installed) to absolute paths at generation time and bakes their +# directories into an explicit PATH, because cron and launchd both run with a +# minimal PATH that will not see a Node version manager's install directory or +# a Homebrew-installed backend CLI. quota-axi and jq are required (the script +# cannot read quota or parse it without them); a backend CLI is optional and +# simply omitted if not found, since which backend(s) are actually in use can +# change over time and any installed one might be needed by a live crewmate. +# Every resolved binary is then re-checked against the generated PATH with a +# scrubbed environment before printing, so a stale or inconsistent PATH is +# caught here instead of failing silently on every cron firing (the exact +# production failure this script now guards against: quota-axi resolved fine +# so quota reading kept working, while herdr did not, so every interrupt/resume +# send failed silently until diagnosed live). If any of these move (a new Node +# version, a reinstall, a newly installed backend), regenerate the line rather +# than hand-editing the stale path. set -eu SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -30,7 +42,7 @@ FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" fm_quota_install_usage() { - sed -n '2,26{s/^# \{0,1\}//;p;}' "$SCRIPT_DIR/fm-quota-watch-install.sh" + sed -n '2,37{s/^# \{0,1\}//;p;}' "$SCRIPT_DIR/fm-quota-watch-install.sh" } MODE=crontab-print @@ -52,14 +64,65 @@ case "$INTERVAL" in ''|*[!0-9]*|0) echo "fm-quota-watch-install.sh: --interval-minutes must be a positive integer, got '$INTERVAL'" >&2; exit 2 ;; esac +# Space-separated, order-preserving, de-duplicated directory list. Args are +# candidate directories (possibly empty/duplicate); each non-empty, not-yet-seen +# one is appended. +fm_quota_install_add_dir() { # + local d=$1 existing + [ -n "$d" ] || return 0 + for existing in $RESOLVED_DIRS; do + [ "$existing" != "$d" ] || return 0 + done + RESOLVED_DIRS="${RESOLVED_DIRS:+$RESOLVED_DIRS }$d" +} + +RESOLVED_DIRS="" +RESOLVED_BINS="" + if ! QUOTA_AXI_BIN=$(command -v quota-axi 2>/dev/null); then echo "fm-quota-watch-install.sh: quota-axi not found on PATH; install/authenticate it first (quota-axi --allow-keychain-prompt), then rerun" >&2 exit 1 fi -QUOTA_AXI_DIR=$(dirname "$QUOTA_AXI_BIN") +fm_quota_install_add_dir "$(dirname "$QUOTA_AXI_BIN")" +RESOLVED_BINS="quota-axi" + +if ! JQ_BIN=$(command -v jq 2>/dev/null); then + echo "fm-quota-watch-install.sh: jq not found on PATH; install it first, then rerun" >&2 + exit 1 +fi +fm_quota_install_add_dir "$(dirname "$JQ_BIN")" +RESOLVED_BINS="$RESOLVED_BINS jq" + +# Optional session-backend CLIs fm-send.sh may dispatch through. Each is +# included only if actually installed; a missing one is silently skipped here +# (its absence is only a real problem if that backend is genuinely in use, and +# fm-send.sh already reports that loudly at send time). +for backend_bin in tmux herdr zellij cmux; do + if bin_path=$(command -v "$backend_bin" 2>/dev/null); then + fm_quota_install_add_dir "$(dirname "$bin_path")" + RESOLVED_BINS="$RESOLVED_BINS $backend_bin" + fi +done + WATCH_BIN="$SCRIPT_DIR/fm-quota-watch.sh" LOG_FILE="$FM_HOME/state/quota-watch.log" -CRON_PATH="/usr/bin:/bin:/usr/sbin:/sbin:$QUOTA_AXI_DIR" +CRON_PATH="/usr/bin:/bin:/usr/sbin:/sbin" +for d in $RESOLVED_DIRS; do + CRON_PATH="$CRON_PATH:$d" +done + +# Prove the generated PATH actually resolves every binary just resolved above, +# in a scrubbed environment matching what cron/launchd hand the script - this +# is the exact class of bug a captain hit in production (quota-axi resolved so +# reading always worked, herdr did not so every send silently failed). +for b in $RESOLVED_BINS; do + if ! env -i PATH="$CRON_PATH" command -v "$b" >/dev/null 2>&1; then + echo "fm-quota-watch-install.sh: internal error: '$b' resolved during generation but not through the generated PATH ('$CRON_PATH'); refusing to print a broken entry" >&2 + exit 1 + fi +done +echo "fm-quota-watch-install.sh: PATH will include: $RESOLVED_BINS" >&2 + CRON_LINE="*/$INTERVAL * * * * PATH=\"$CRON_PATH\" FM_HOME=\"$FM_HOME\" \"$WATCH_BIN\" >> \"$LOG_FILE\" 2>&1" case "$MODE" in diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 884546233a..b5d970d163 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -125,6 +125,7 @@ family_for_basename() { fm-kimi-harness.test.sh|fm-herdr-lab.test.sh|fm-instruction-owners.test.sh|fm-lint.test.sh|\ fm-install-herdr.test.sh|fm-nm-test-contract.test.sh|fm-no-mistakes-ownership.test.sh|\ fm-operational-input.test.sh|fm-pi-primary-types.test.sh|fm-quota-watch.test.sh|\ + fm-quota-watch-install.test.sh|\ fm-send-popup-settle.test.sh|fm-send-settle.test.sh|fm-stow-contract.test.sh|\ fm-subagent-pretool-check.test.sh|\ fm-supervision-instructions.test.sh|fm-tmux-submit-busy.test.sh|fm-transition-lib.test.sh|\ diff --git a/docs/quota-watch.md b/docs/quota-watch.md index 95af41d4a8..3bd8ab86be 100644 --- a/docs/quota-watch.md +++ b/docs/quota-watch.md @@ -93,9 +93,24 @@ or pulling firstmate changes nothing about your machine's scheduler. Run `bin/fm-quota-watch-install.sh` yourself once to see the exact line, or add `--install-crontab` to append it directly (idempotent: it refuses if an entry for this script is already present). -It resolves `quota-axi` to an absolute path and bakes an explicit `PATH` into -the generated entry, because both `cron` and `launchd` run with a minimal -`PATH` that will not see a Node version manager's install directory on its own. +It resolves `quota-axi`, `jq`, and every session-backend CLI it finds +installed (`tmux`, `herdr`, `zellij`, `cmux`) to absolute paths and bakes an +explicit `PATH` into the generated entry, because both `cron` and `launchd` +run with a minimal `PATH` that will not see a Node version manager's install +directory or a Homebrew-installed backend CLI on its own. +`quota-axi` and `jq` are required (the script cannot function without them, +so a missing one is a hard failure); a backend CLI is included only if it is +actually installed, since `fm-quota-watch.sh` calls `bin/fm-send.sh`, which +dispatches through whichever backend a live crewmate happens to use. +Every resolved binary is then re-checked against the generated PATH under a +scrubbed environment before printing, so a stale or inconsistent PATH is +caught here rather than failing silently on every cron firing - the real +production failure this guards against: `quota-axi` resolved fine (so quota +*reading* always worked), while `herdr` lived in a directory the generated +PATH omitted, so every interrupt/resume *send* through `fm-send.sh` failed +silently until diagnosed live. +The printed `PATH will include: ...` line names exactly what was found - +double-check it names every backend you actually use before installing. ```sh bin/fm-quota-watch-install.sh # print a crontab line @@ -106,10 +121,6 @@ bin/fm-quota-watch-install.sh --launchd # print a launchd plist instead A 5-minute default cadence is frequent enough to catch a crossing quickly relative to the shortest quota window (the multi-hour session window) without meaningfully adding to system load; `--interval-minutes` overrides it. -If your session backend needs its own CLI on `PATH` (`tmux`, `herdr`, -`zellij`, ...), extend the printed `PATH` value the same way before installing -it, since `fm-quota-watch.sh` calls `bin/fm-send.sh`, which dispatches through -that backend. `quota-axi` itself needs Claude auth once per machine: if `--status` (or a real run) reports no usable reading and `quota-axi --json` shows @@ -127,3 +138,10 @@ reruns, picking up crew spawned mid-pause, the hysteresis band, recovery, `credits` reading never driving a pause or blocking a resume while `session`/`weekly` stay low, all against fixture quota JSON and a fake sender - no real quota reading, backend, or live fleet involved. + +`tests/fm-quota-watch-install.test.sh` covers the install script itself: every +resolved binary (`quota-axi`, `jq`, and whichever of `tmux`/`herdr` are present) +actually resolves through the generated crontab and launchd `PATH` under a +fully scrubbed environment - the regression coverage for the silent-send- +failure production bug above - plus a not-installed backend CLI being omitted +without error, and missing `quota-axi`/`jq` each being a clear hard failure. diff --git a/tests/fm-quota-watch-install.test.sh b/tests/fm-quota-watch-install.test.sh new file mode 100755 index 0000000000..633d75dcaf --- /dev/null +++ b/tests/fm-quota-watch-install.test.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Tests for bin/fm-quota-watch-install.sh: the print-only (by default) crontab/ +# launchd generator for bin/fm-quota-watch.sh. +# +# Regression target: a production bug where the generated PATH resolved +# quota-axi (so quota reading always worked) but not herdr (installed in a +# different directory), so every interrupt/resume send through fm-send.sh +# failed silently on every cron firing. Every case here fully controls PATH +# (no reliance on whatever happens to be installed on the machine running the +# suite) via small per-binary fake directories, and proves - the same way a +# real cron/launchd invocation would see it - that the generated PATH actually +# resolves every binary it claims to include. +# +# Matrix: +# (a) quota-axi, jq, and every found optional backend CLI (here: tmux and +# herdr, each in its own directory) all resolve through the generated +# crontab-line PATH under a fully scrubbed environment +# (b) the same holds for the generated launchd plist's PATH +# (c) a backend CLI that is not installed (zellij, cmux) is silently +# omitted - no error, and it is not falsely claimed as resolvable +# (d) missing quota-axi is a hard, clear failure +# (e) missing jq is a hard, clear failure +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +INSTALL_BIN="$ROOT/bin/fm-quota-watch-install.sh" +TMP_ROOT=$(fm_test_tmproot fm-quota-watch-install-tests) + +# One fake executable named , alone in its own directory. Echoes the +# directory so callers can build a PATH from several of these to test +# multi-directory merging without polluting a shared fakebin. +make_fake_bin_dir() { # + local case_dir=$1 name=$2 dir + dir="$case_dir/bin-$name" + mkdir -p "$dir" + cat > "$dir/$name" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + chmod +x "$dir/$name" + printf '%s\n' "$dir" +} + +# A directory of symlinks to exactly the coreutils fm-quota-watch-install.sh +# and its bash shebang need (bash itself, cat, dirname, env, grep, sed) - +# resolved from THIS test run's own real PATH - and nothing else. Used only by +# the missing-quota-axi/missing-jq cases so a real system jq (observed at +# /usr/bin/jq on some machines) cannot mask the "jq is missing" scenario the +# way a plain /usr/bin:/bin:/usr/sbin:/sbin prefix would. +make_curated_system_dir() { # + local case_dir=$1 dir tool resolved + dir="$case_dir/system" + mkdir -p "$dir" + for tool in bash cat dirname env grep sed; do + resolved=$(command -v "$tool") || fail "test setup: '$tool' not found on this machine's own PATH" + ln -s "$resolved" "$dir/$tool" + done + printf '%s\n' "$dir" +} + +# Extract the PATH="..." value from a generated crontab line. +extract_cron_path() { # + printf '%s\n' "$1" | grep -o 'PATH="[^"]*"' | head -1 | sed 's/^PATH="//;s/"$//' +} + +# Extract the PATH string value from a generated launchd plist (the +# immediately following the PATH ). +extract_launchd_path() { # + printf '%s\n' "$1" | awk ' + /PATH<\/key>/ { want=1; next } + want { gsub(/<\/?string>/, ""); gsub(/^[ \t]+|[ \t]+$/, ""); print; exit } + ' +} + +# --- (a)/(c)/(d)/(e) crontab generation --------------------------------------- + +test_crontab_path_resolves_every_included_binary() { + local case_dir d_quota d_jq d_tmux d_herdr full_path out rc cron_line generated_path + case_dir="$TMP_ROOT/${FUNCNAME[0]}" + mkdir -p "$case_dir" + d_quota=$(make_fake_bin_dir "$case_dir" quota-axi) + d_jq=$(make_fake_bin_dir "$case_dir" jq) + d_tmux=$(make_fake_bin_dir "$case_dir" tmux) + d_herdr=$(make_fake_bin_dir "$case_dir" herdr) + full_path="$d_quota:$d_jq:$d_tmux:$d_herdr:/usr/bin:/bin:/usr/sbin:/sbin" + + out=$(PATH="$full_path" FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$case_dir/home" "$INSTALL_BIN" 2>&1) + rc=$? + expect_code 0 "$rc" "crontab generation succeeds when quota-axi/jq/tmux/herdr are all present" + + cron_line=$(printf '%s\n' "$out" | grep -F '*/5 * * * *') + [ -n "$cron_line" ] || fail "no crontab line found in output: $out" + generated_path=$(extract_cron_path "$cron_line") + [ -n "$generated_path" ] || fail "could not extract PATH= from generated crontab line" + + for bin in quota-axi jq tmux herdr; do + env -i PATH="$generated_path" command -v "$bin" >/dev/null 2>&1 \ + || fail "generated crontab PATH does not resolve '$bin' the way a real cron invocation would (PATH=$generated_path)" + done + + # zellij/cmux were never installed anywhere on $full_path: the generated + # PATH must not claim to resolve them either - a false resolution here would + # mean the install script fabricated a directory it never actually checked. + for bin in zellij cmux; do + if env -i PATH="$generated_path" command -v "$bin" >/dev/null 2>&1; then + fail "generated crontab PATH unexpectedly resolves '$bin', which was never installed on the generation PATH" + fi + done + + assert_contains "$out" "PATH will include: quota-axi jq tmux herdr" "install script reports exactly the binaries it found" + + pass "crontab-generated PATH resolves quota-axi/jq/tmux/herdr and omits zellij/cmux, verified under a scrubbed environment" +} + +test_launchd_path_resolves_every_included_binary() { + local case_dir d_quota d_jq d_tmux d_herdr full_path out rc generated_path + case_dir="$TMP_ROOT/${FUNCNAME[0]}" + mkdir -p "$case_dir" + d_quota=$(make_fake_bin_dir "$case_dir" quota-axi) + d_jq=$(make_fake_bin_dir "$case_dir" jq) + d_tmux=$(make_fake_bin_dir "$case_dir" tmux) + d_herdr=$(make_fake_bin_dir "$case_dir" herdr) + full_path="$d_quota:$d_jq:$d_tmux:$d_herdr:/usr/bin:/bin:/usr/sbin:/sbin" + + out=$(PATH="$full_path" FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$case_dir/home" "$INSTALL_BIN" --launchd 2>&1) + rc=$? + expect_code 0 "$rc" "launchd generation succeeds when quota-axi/jq/tmux/herdr are all present" + + generated_path=$(extract_launchd_path "$out") + [ -n "$generated_path" ] || fail "could not extract the PATH string from the generated launchd plist" + + for bin in quota-axi jq tmux herdr; do + env -i PATH="$generated_path" command -v "$bin" >/dev/null 2>&1 \ + || fail "generated launchd PATH does not resolve '$bin' the way launchd's own scrubbed environment would (PATH=$generated_path)" + done + + pass "launchd-generated PATH resolves quota-axi/jq/tmux/herdr under a scrubbed environment" +} + +test_missing_quota_axi_is_a_hard_failure() { + local case_dir d_jq d_system out rc + case_dir="$TMP_ROOT/${FUNCNAME[0]}" + mkdir -p "$case_dir" + d_jq=$(make_fake_bin_dir "$case_dir" jq) + d_system=$(make_curated_system_dir "$case_dir") + + out=$(PATH="$d_jq:$d_system" FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$case_dir/home" "$INSTALL_BIN" 2>&1) + rc=$? + [ "$rc" -ne 0 ] || fail "install script must refuse to generate an entry when quota-axi is missing" + assert_contains "$out" "quota-axi not found on PATH" "clear message when quota-axi is missing" + + pass "missing quota-axi is a hard, clear failure rather than a broken generated entry" +} + +test_missing_jq_is_a_hard_failure() { + local case_dir d_quota d_system out rc + case_dir="$TMP_ROOT/${FUNCNAME[0]}" + mkdir -p "$case_dir" + d_quota=$(make_fake_bin_dir "$case_dir" quota-axi) + d_system=$(make_curated_system_dir "$case_dir") + + out=$(PATH="$d_quota:$d_system" FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$case_dir/home" "$INSTALL_BIN" 2>&1) + rc=$? + [ "$rc" -ne 0 ] || fail "install script must refuse to generate an entry when jq is missing" + assert_contains "$out" "jq not found on PATH" "clear message when jq is missing" + + pass "missing jq is a hard, clear failure rather than a broken generated entry" +} + +# --- run ----------------------------------------------------------------- + +test_crontab_path_resolves_every_included_binary +test_launchd_path_resolves_every_included_binary +test_missing_quota_axi_is_a_hard_failure +test_missing_jq_is_a_hard_failure From 6475448addbc396b3082d0c0fbc6dc65b147d991 Mon Sep 17 00:00:00 2001 From: Jair Date: Wed, 29 Jul 2026 17:54:26 -0300 Subject: [PATCH 6/9] no-mistakes(review): fix(quota-watch): retry resume send failures instead of dropping pause tracking --- bin/fm-quota-watch.sh | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/bin/fm-quota-watch.sh b/bin/fm-quota-watch.sh index 56a8cf813a..7146ae4d43 100755 --- a/bin/fm-quota-watch.sh +++ b/bin/fm-quota-watch.sh @@ -266,7 +266,17 @@ fm_quota_watch_pause() { } fm_quota_watch_resume() { - local pct=$1 id meta acted=0 + local pct=$1 id meta acted=0 pending=0 paused_at prior_paused_at tmp_flag + paused_at=$(date +%s) + prior_paused_at=$(fm_meta_get "$PAUSE_FLAG" paused_at) + [ -n "$prior_paused_at" ] && paused_at=$prior_paused_at + + tmp_flag=$(mktemp "${TMPDIR:-/tmp}/fm-quota-paused.XXXXXX") || return 1 + { + printf 'paused_at=%s\n' "$paused_at" + printf 'pct=%s\n' "$pct" + } > "$tmp_flag" + while IFS= read -r id; do [ -n "$id" ] || continue meta="$STATE/$id.meta" @@ -277,11 +287,19 @@ fm_quota_watch_resume() { if "$FM_QUOTA_SEND_BIN" "$id" "Quota recovered (now ${pct}% used, below the ${RESUME_THRESHOLD}% resume threshold). Continue where you left off."; then acted=$((acted + 1)) else - echo "fm-quota-watch.sh: warning: could not deliver the resume message to $id" >&2 + echo "fm-quota-watch.sh: warning: could not deliver the resume message to $id; keeping it recorded as paused for a retry" >&2 + printf 'task=%s\n' "$id" >> "$tmp_flag" + pending=$((pending + 1)) fi done < <(fm_quota_watch_flag_tasks) - rm -f "$PAUSE_FLAG" - echo "fm-quota-watch.sh: pct=$pct < resume threshold $RESUME_THRESHOLD - resumed $acted crew, cleared the pause" + + if [ "$pending" -gt 0 ]; then + mv "$tmp_flag" "$PAUSE_FLAG" + echo "fm-quota-watch.sh: pct=$pct < resume threshold $RESUME_THRESHOLD - resumed $acted crew, $pending still pending a retry" + else + rm -f "$tmp_flag" "$PAUSE_FLAG" + echo "fm-quota-watch.sh: pct=$pct < resume threshold $RESUME_THRESHOLD - resumed $acted crew, cleared the pause" + fi } # --- act ----------------------------------------------------------------- From 2ef5413e619e268e7b3909fc06d0421da7b43b10 Mon Sep 17 00:00:00 2001 From: Jair Date: Wed, 29 Jul 2026 17:57:47 -0300 Subject: [PATCH 7/9] no-mistakes(document): docs(quota-watch): describe resume retry-on-failed-delivery behavior --- docs/quota-watch.md | 8 ++++++-- tests/fm-quota-watch.test.sh | 39 ++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/docs/quota-watch.md b/docs/quota-watch.md index 3bd8ab86be..f53b9ab500 100644 --- a/docs/quota-watch.md +++ b/docs/quota-watch.md @@ -31,8 +31,12 @@ Once per invocation: read Claude's current usage, take the highest - **Below the resume threshold** (default 65%, deliberately lower than the pause threshold so a reading oscillating near 80% does not flap crew back and forth): send one short note to every crewmate this script paused and - clear the flag. A crewmate that declared its own unrelated `paused:` wait is - left alone - only crew recorded in `state/.quota-paused` are resumed. + clear the flag once every note delivers. A crewmate that declared its own + unrelated `paused:` wait is left alone - only crew recorded in + `state/.quota-paused` are resumed. If a note fails to deliver, that + crewmate stays recorded in the flag for a retry on the next run instead of + being dropped from tracking; crew that already received their note are not + re-notified on the retry. - **Between the two thresholds, or below the pause threshold with nothing paused**: no-op. - **No usable reading** - `quota-axi` missing, `auth_required`, or malformed diff --git a/tests/fm-quota-watch.test.sh b/tests/fm-quota-watch.test.sh index ea94d203c7..c18f27189a 100755 --- a/tests/fm-quota-watch.test.sh +++ b/tests/fm-quota-watch.test.sh @@ -107,6 +107,7 @@ run_watch() { FM_TEST_SEND_LOG="$case_dir/send.log" \ FM_QUOTA_PAUSE_THRESHOLD=80 \ FM_QUOTA_RESUME_THRESHOLD=65 \ + FM_TEST_SEND_FAIL_ID="${FM_TEST_SEND_FAIL_ID:-}" \ "$QUOTA_WATCH" "$@" > "$OUT" 2> "$ERR" RC=$? } @@ -360,6 +361,43 @@ test_credits_window_ignored_during_resume_decision() { pass "recovery is decided from session/weekly alone, even while the credits window stays high" } +# --- (k) resume retries a failed send instead of dropping the pause ----------- + +test_resume_retries_failed_send() { + local case_dir + case_dir=$(make_case resume-retry) + write_crew_meta "$case_dir" task-a ship claude + write_crew_meta "$case_dir" task-b scout opencode + quota_json_pct 88 > "$case_dir/quota.json" + run_watch "$case_dir" + expect_code 0 "$RC" "pause run exits 0" + assert_present "$case_dir/state/.quota-paused" "flag created on cross" + + quota_json_pct 50 > "$case_dir/quota.json" + FM_TEST_SEND_FAIL_ID=task-b run_watch "$case_dir" + expect_code 0 "$RC" "resume run (one delivery failure) exits 0" + + assert_present "$case_dir/state/.quota-paused" "flag is kept when a resume note fails to deliver" + assert_no_grep "task=task-a" "$case_dir/state/.quota-paused" "successfully-notified crew is dropped from the retry flag" + assert_grep "task=task-b" "$case_dir/state/.quota-paused" "failed-delivery crew stays recorded for a retry" + assert_grep "task-a Quota recovered" "$case_dir/send.log" "task-a still got its resume note" + + local task_b_attempts_first + task_b_attempts_first=$(grep -c 'task-b Quota recovered' "$case_dir/send.log") + [ "$task_b_attempts_first" -eq 1 ] || fail "expected exactly 1 delivery attempt to task-b, got $task_b_attempts_first" + + run_watch "$case_dir" + expect_code 0 "$RC" "retry run (delivery now succeeds) exits 0" + assert_absent "$case_dir/state/.quota-paused" "flag is cleared once the retried note is delivered" + local task_a_total task_b_total + task_a_total=$(grep -c 'task-a Quota recovered' "$case_dir/send.log") + task_b_total=$(grep -c 'task-b Quota recovered' "$case_dir/send.log") + [ "$task_a_total" -eq 1 ] || fail "task-a resume note was resent on retry ($task_a_total deliveries)" + [ "$task_b_total" -eq 2 ] || fail "expected exactly 2 delivery attempts to task-b (1 failed + 1 retried), got $task_b_total" + + pass "resume retries a failed send instead of dropping pause tracking, without re-notifying already-resumed crew" +} + # --- run ----------------------------------------------------------------- test_pause_crosses_threshold @@ -373,3 +411,4 @@ test_unknown_harness_refuses_to_guess test_status_flag_smoke test_credits_window_never_drives_pause test_credits_window_ignored_during_resume_decision +test_resume_retries_failed_send From 9c2b8cbec843c3df7fc512e8daa3af125beb6ac9 Mon Sep 17 00:00:00 2001 From: Jair Date: Wed, 29 Jul 2026 18:51:51 -0300 Subject: [PATCH 8/9] no-mistakes: apply CI fixes --- bin/fm-quota-watch-install.sh | 2 +- tests/fm-quota-watch-install.test.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/fm-quota-watch-install.sh b/bin/fm-quota-watch-install.sh index 289cb13669..73e387fe3f 100755 --- a/bin/fm-quota-watch-install.sh +++ b/bin/fm-quota-watch-install.sh @@ -116,7 +116,7 @@ done # is the exact class of bug a captain hit in production (quota-axi resolved so # reading always worked, herdr did not so every send silently failed). for b in $RESOLVED_BINS; do - if ! env -i PATH="$CRON_PATH" command -v "$b" >/dev/null 2>&1; then + if ! env -i PATH="$CRON_PATH" sh -c 'command -v "$1"' _ "$b" >/dev/null 2>&1; then echo "fm-quota-watch-install.sh: internal error: '$b' resolved during generation but not through the generated PATH ('$CRON_PATH'); refusing to print a broken entry" >&2 exit 1 fi diff --git a/tests/fm-quota-watch-install.test.sh b/tests/fm-quota-watch-install.test.sh index 633d75dcaf..e2a0378608 100755 --- a/tests/fm-quota-watch-install.test.sh +++ b/tests/fm-quota-watch-install.test.sh @@ -96,7 +96,7 @@ test_crontab_path_resolves_every_included_binary() { [ -n "$generated_path" ] || fail "could not extract PATH= from generated crontab line" for bin in quota-axi jq tmux herdr; do - env -i PATH="$generated_path" command -v "$bin" >/dev/null 2>&1 \ + env -i PATH="$generated_path" sh -c 'command -v "$1"' _ "$bin" >/dev/null 2>&1 \ || fail "generated crontab PATH does not resolve '$bin' the way a real cron invocation would (PATH=$generated_path)" done @@ -104,7 +104,7 @@ test_crontab_path_resolves_every_included_binary() { # PATH must not claim to resolve them either - a false resolution here would # mean the install script fabricated a directory it never actually checked. for bin in zellij cmux; do - if env -i PATH="$generated_path" command -v "$bin" >/dev/null 2>&1; then + if env -i PATH="$generated_path" sh -c 'command -v "$1"' _ "$bin" >/dev/null 2>&1; then fail "generated crontab PATH unexpectedly resolves '$bin', which was never installed on the generation PATH" fi done @@ -132,7 +132,7 @@ test_launchd_path_resolves_every_included_binary() { [ -n "$generated_path" ] || fail "could not extract the PATH string from the generated launchd plist" for bin in quota-axi jq tmux herdr; do - env -i PATH="$generated_path" command -v "$bin" >/dev/null 2>&1 \ + env -i PATH="$generated_path" sh -c 'command -v "$1"' _ "$bin" >/dev/null 2>&1 \ || fail "generated launchd PATH does not resolve '$bin' the way launchd's own scrubbed environment would (PATH=$generated_path)" done From 0aab61f9476b2dc88ef919be2fd798617191f1f7 Mon Sep 17 00:00:00 2001 From: Jair Date: Wed, 29 Jul 2026 19:41:17 -0300 Subject: [PATCH 9/9] no-mistakes: apply CI fixes --- bin/fm-quota-watch-install.sh | 1 + tests/fm-quota-watch-install.test.sh | 3 +++ 2 files changed, 4 insertions(+) diff --git a/bin/fm-quota-watch-install.sh b/bin/fm-quota-watch-install.sh index 73e387fe3f..fa817d9b91 100755 --- a/bin/fm-quota-watch-install.sh +++ b/bin/fm-quota-watch-install.sh @@ -116,6 +116,7 @@ done # is the exact class of bug a captain hit in production (quota-axi resolved so # reading always worked, herdr did not so every send silently failed). for b in $RESOLVED_BINS; do + # shellcheck disable=SC2016 # intentional: $1 expands in the inner sh -c, not here. if ! env -i PATH="$CRON_PATH" sh -c 'command -v "$1"' _ "$b" >/dev/null 2>&1; then echo "fm-quota-watch-install.sh: internal error: '$b' resolved during generation but not through the generated PATH ('$CRON_PATH'); refusing to print a broken entry" >&2 exit 1 diff --git a/tests/fm-quota-watch-install.test.sh b/tests/fm-quota-watch-install.test.sh index e2a0378608..8fe0146a59 100755 --- a/tests/fm-quota-watch-install.test.sh +++ b/tests/fm-quota-watch-install.test.sh @@ -96,6 +96,7 @@ test_crontab_path_resolves_every_included_binary() { [ -n "$generated_path" ] || fail "could not extract PATH= from generated crontab line" for bin in quota-axi jq tmux herdr; do + # shellcheck disable=SC2016 # intentional: $1 expands in the inner sh -c, not here. env -i PATH="$generated_path" sh -c 'command -v "$1"' _ "$bin" >/dev/null 2>&1 \ || fail "generated crontab PATH does not resolve '$bin' the way a real cron invocation would (PATH=$generated_path)" done @@ -104,6 +105,7 @@ test_crontab_path_resolves_every_included_binary() { # PATH must not claim to resolve them either - a false resolution here would # mean the install script fabricated a directory it never actually checked. for bin in zellij cmux; do + # shellcheck disable=SC2016 # intentional: $1 expands in the inner sh -c, not here. if env -i PATH="$generated_path" sh -c 'command -v "$1"' _ "$bin" >/dev/null 2>&1; then fail "generated crontab PATH unexpectedly resolves '$bin', which was never installed on the generation PATH" fi @@ -132,6 +134,7 @@ test_launchd_path_resolves_every_included_binary() { [ -n "$generated_path" ] || fail "could not extract the PATH string from the generated launchd plist" for bin in quota-axi jq tmux herdr; do + # shellcheck disable=SC2016 # intentional: $1 expands in the inner sh -c, not here. env -i PATH="$generated_path" sh -c 'command -v "$1"' _ "$bin" >/dev/null 2>&1 \ || fail "generated launchd PATH does not resolve '$bin' the way launchd's own scrubbed environment would (PATH=$generated_path)" done