From 50cc24a7d975ee3a066bb16f448e60c8e0bc684d Mon Sep 17 00:00:00 2001 From: ItsFlow Date: Wed, 22 Jul 2026 19:30:35 +0100 Subject: [PATCH 01/34] fix: block primary-session delegation outside the fleet (#854) * feat: fence primary-session delegation outside the fleet A firstmate primary that delegates through Claude Code's built-in delegation tools creates work with no state/.meta. Because fm-supervision-lib.sh counts *.meta and fm-turnend-guard.sh exits silently at zero, such work does not merely go unsupervised: it makes the whole guard stack structurally inert, and it dies with the primary session. On 2026-07-22 that cost two workers mid-flight and left supervision down for 73 minutes unnoticed. Layer 1, the primary fix: a permissions.deny list in .claude/settings.json removes the 18 delegation, scheduling, worktree, and task-tracking tools from the model's schema, so they are never offered. This is removal rather than interception, so there is no call to intercept and no fail-open path. The list is flat and in one file so its width stays reviewable; the captain owns that width. Layer 2, bin/fm-subagent-pretool-check.sh: a deny list is fail-open against tools that do not exist yet, and permissions.allow is a pre-approval list rather than an availability list, so there is no fail-closed allowlist to use instead. This backstop classifies the tool NAME by shape rather than against a fixed list, so a delegation tool that ships before the deny list is updated is still refused. It excludes mcp__* names and observe-or-stop operations, scopes itself to a genuine primary home via the shared fm_primary_scope_matches predicate so a crewmate's task worktree is unaffected, and offers one deliberate FM_ALLOW_SUBAGENT=1 escape hatch that must be set at launch. Verified live against Claude Code 2.1.217, including a deny-key A/B with a nonsense-name control, layer 2 denying an un-denied Workflow call, the same call allowed in a linked worktree, and the escape hatch. Corrects a prior finding: both Task and Agent work as deny keys, so both are pinned. Codex 0.144.1 verified to expose no delegation tool; grok, opencode, and pi are inspected and documented as not wired because those binaries are absent from this host and the repo requires live validation before trusting a harness hook. Evidence in docs/subagent-guard.md. * no-mistakes(review): Ship scoped Claude delegation guard * no-mistakes(test): Ship Claude delegation deny list * no-mistakes(document): Clarify PreToolUse guard ownership * no-mistakes(lint): Keep Claude deny list local --- .agents/skills/harness-adapters/SKILL.md | 13 +- .claude/settings.json | 9 + bin/fm-subagent-pretool-check.sh | 194 ++++++++++++ bin/fm-test-run.sh | 1 + docs/scripts.md | 1 + docs/subagent-guard.md | 367 +++++++++++++++++++++++ docs/turnend-guard.md | 4 +- tests/fm-subagent-pretool-check.test.sh | 284 ++++++++++++++++++ 8 files changed, 870 insertions(+), 3 deletions(-) create mode 100755 bin/fm-subagent-pretool-check.sh create mode 100644 docs/subagent-guard.md create mode 100755 tests/fm-subagent-pretool-check.test.sh diff --git a/.agents/skills/harness-adapters/SKILL.md b/.agents/skills/harness-adapters/SKILL.md index 761f07d..b208547 100644 --- a/.agents/skills/harness-adapters/SKILL.md +++ b/.agents/skills/harness-adapters/SKILL.md @@ -62,7 +62,18 @@ Every verified primary harness also has a wired PreToolUse-equivalent hook that `claude` and `codex` block directly through PreToolUse hooks; `grok` blocks the same way but requires every `$VAR` reference in its hook `command` string to carry an inline `:-default` or it fails to launch the hook entirely. `opencode` and `pi` block by throwing from `tool.execute.before` / returning `{block: true}` from `tool_call`. The exact hook files, commands, output-shaping quirks (Claude Code only honors the deny when stdout is empty), and validation transcripts are owned by `docs/arm-pretool-check.md`. -When changing any primary PreToolUse hook, validate the real harness behavior in a scratch project before trusting it, then update that doc. +When changing any watcher-arm PreToolUse hook, validate the real harness behavior in a scratch project before trusting it, then update that doc. +## Primary delegation-shape guard + +Claude exposes built-in delegation, scheduling, and worktree tools that a primary session can use to create work with no `state/.meta`, which makes the whole guard stack inert because every guard counts that metadata. +The shipped mechanism is `bin/fm-subagent-pretool-check.sh`, a primary-home PreToolUse guard that denies a delegation-SHAPED tool name. +Claude primaries should also use an untracked per-home local `permissions.deny` list as hardening for known Claude delegation tools, because it removes them from the model's schema so they are never offered. +That deny list must not ship in tracked `.claude/settings.json` because it is Claude-only rather than harness-agnostic, and because tracked project settings propagate into linked worktrees where they disarm legitimate crewmates. +`docs/subagent-guard.md` owns the full contract, the local deny-list recommendation, the `FM_ALLOW_SUBAGENT=1` escape hatch, and the per-harness applicability review. + +Two verified facts worth pinning here. +The subagent tool presents to the model as `Agent`, and on Claude Code 2.1.217 both `Agent` and `Task` work as `permissions.deny` keys, verified by an A/B with a nonsense-name control. +`permissions.allow` is a pre-approval list rather than an availability list, so there is no fail-closed positive allowlist. ## Primary session-start nudge diff --git a/.claude/settings.json b/.claude/settings.json index 37a535e..4ad0e1a 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -28,6 +28,15 @@ "command": "\"$CLAUDE_PROJECT_DIR\"/bin/fm-continuity-pretool-check.sh" } ] + }, + { + "matcher": ".*", + "hooks": [ + { + "type": "command", + "command": "\"$CLAUDE_PROJECT_DIR\"/bin/fm-subagent-pretool-check.sh --claude" + } + ] } ], "Stop": [ diff --git a/bin/fm-subagent-pretool-check.sh b/bin/fm-subagent-pretool-check.sh new file mode 100755 index 0000000..d179ded --- /dev/null +++ b/bin/fm-subagent-pretool-check.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# PreToolUse guard against primary-session delegation outside the fleet. +# +# A firstmate primary that delegates through a harness's own delegation, +# scheduling, or background-work tool creates work with no `state/.meta` and +# no `data//brief.md`. Only `bin/fm-spawn.sh` writes that metadata, and +# every firstmate guard keys off it (bin/fm-supervision-lib.sh counts +# `state/*.meta`; bin/fm-turnend-guard.sh exits silently at zero). So such work +# is not merely unsupervised: it makes the whole guard stack structurally inert, +# and it dies with the primary session instead of living in its own backend +# session. +# +# This scoped PreToolUse guard is the shipped mechanism. +# Claude primaries should also use an untracked per-home local +# `permissions.deny` list as hardening for known Claude delegation tools, +# because it removes them from the model's schema entirely. +# That deny list must not be tracked: it is Claude-only rather than +# harness-agnostic, and tracked project settings propagate into linked +# worktrees where they disarm legitimate crewmates. +# The tracked Claude matcher is deliberately `.*`: a stem-enumerating matcher +# would reintroduce the fail-open-by-enumeration problem this guard exists to +# solve, because any future tool name outside the matcher would never reach this +# script. +# This script is therefore the single owner of classification. +# It matches a delegation-SHAPED tool name rather than a fixed list, so a future +# tool that ships before anyone updates a local deny list is still refused. +# +# The guard is narrow by design. It classifies ONE thing: the shape of the tool +# name. It makes no judgment about whether the work should be delegated at all, +# which is a reasoning boundary no tool-shape hook can enforce. +# See docs/subagent-guard.md for the complete contract and validation record. +# +# Usage: +# | bin/fm-subagent-pretool-check.sh +# bin/fm-subagent-pretool-check.sh --tool '' +# +# Stdin mode extracts .tool_name for Claude and Codex, or .toolName for Grok. +# CLI mode is for adapters that already hold the tool name (OpenCode, Pi). +# +# Exit/output contract (identical shape to bin/fm-cd-pretool-check.sh): +# ALLOW - exit 0 and no output. +# DENY - exit 2, a Claude-shaped deny object on stderr, and a Grok-shaped +# deny object on stdout unless --claude was supplied. +# INERT - not a genuine primary home (a crewmate/scout task worktree or a +# non-firstmate repo): exit 0 with no output, exactly like ALLOW. +# ESCAPE - FM_ALLOW_SUBAGENT=1 in the environment allows deliberately. +# FAIL OPEN - malformed or empty stdin, or missing jq for stdin transport. +# +# Claude requires stdout to remain empty on deny. +# Codex blocks on exit 2 and displays stderr. +# Grok consumes the stdout decision object. +# OpenCode and Pi consume exit 2 plus stderr. +set -u + +# Lowercase substrings that mark a tool name as delegation-shaped: it creates +# work, an agent, a schedule, or an isolated workspace that firstmate would not +# know about. This list is the single owner of the shipped classification. +DELEGATION_STEMS='agent subagent task workflow cron schedul worktree delegate spawn dispatch handoff remote sendmessage monitor' + +# Exact lowercase tool names that match a stem above but only OBSERVE or STOP +# work that already exists. Reading or ending unaccounted work is not creating +# it, and denying these would strand already-running work with no way to inspect +# or end it. A local Claude deny list may still remove these from the +# schema; this shipped guard deliberately stays narrower so it can never be the +# reason a runaway task cannot be stopped. +OBSERVE_ONLY_TOOLS='taskoutput taskstop taskget tasklist cronlist bashoutput killshell' + +TOOL="" +TOOL_SET=0 +CLAUDE_MODE=0 + +usage() { + cat <<'EOF' +Usage: fm-subagent-pretool-check.sh [--tool ] [--claude] + +With no --tool, reads a PreToolUse-style JSON payload on stdin (Claude/Codex +tool_name, or Grok toolName). +Denies a delegation-SHAPED tool name in a genuine primary home. +Claude primaries may also add an untracked per-home permissions.deny list that +removes known delegation tools from the model schema before this hook is needed. +Do not ship that Claude-only list in tracked project settings, because linked +worktrees inherit it and legitimate crewmates would lose their delegation tools. +This hook remains as the shipped guard for future delegation-shaped names +outside any local fixed list. +Fires only in a genuine firstmate primary home; it is a silent no-op in a +crewmate/scout task worktree or any non-firstmate repo, where a worker using +delegation tools is legitimate. +Exits 0 to allow and 2 to deny, naming the real crewmate dispatch path instead. +Set FM_ALLOW_SUBAGENT=1 in the session environment to allow deliberately. +Malformed transport fails open. +EOF +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --tool) + [ "$#" -gt 1 ] || { echo "error: --tool requires a value" >&2; exit 2; } + TOOL=$2 + TOOL_SET=1 + shift 2 + ;; + --tool=*) + TOOL=${1#--tool=} + TOOL_SET=1 + shift + ;; + --claude) + CLAUDE_MODE=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "error: unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [ "$TOOL_SET" -eq 0 ]; then + PAYLOAD=$(cat 2>/dev/null || true) + [ -n "$PAYLOAD" ] || exit 0 + command -v jq >/dev/null 2>&1 || exit 0 + TOOL=$(printf '%s' "$PAYLOAD" | jq -r '(.tool_name // .toolName // empty)' 2>/dev/null) || exit 0 +fi + +[ -n "$TOOL" ] || exit 0 + +LC_ALL=C NORMALIZED=$(printf '%s' "$TOOL" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9') + +# An MCP tool belongs to an external integration, not to the harness's own +# delegation surface, and its name is chosen by that server. Never classify one +# here: an MCP server with a task or agent noun in a tool name is common and +# blocking it would be a false positive with no bearing on fleet dispatch. +case "$TOOL" in + mcp__*) exit 0 ;; +esac + +for allowed in $OBSERVE_ONLY_TOOLS; do + [ "$NORMALIZED" != "$allowed" ] || exit 0 +done + +MATCHED="" +for stem in $DELEGATION_STEMS; do + case "$NORMALIZED" in + *"$stem"*) MATCHED=$stem; break ;; + esac +done +[ -n "$MATCHED" ] || exit 0 + +# The single deliberate escape hatch. It is an environment variable rather than +# a flag or a state file so it must be set when the session is launched, which +# makes a genuinely intended use possible and an accidental one impossible: no +# in-session tool call can set it for the call that follows. +[ "${FM_ALLOW_SUBAGENT:-}" != "1" ] || exit 0 + +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")" 2>/dev/null && pwd -P) || exit 0 +FM_ROOT=${FM_ROOT_OVERRIDE:-$(CDPATH='' cd -- "$SCRIPT_DIR/.." 2>/dev/null && pwd -P)} || exit 0 +FM_HOME=${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}} +STATE=${FM_STATE_OVERRIDE:-$FM_HOME/state} + +# Scope to a genuine primary home, exactly as the session-start nudge and the +# turn-end guard do. fm_primary_scope_matches accepts a plain checkout or a +# marked secondmate home - both operate a fleet and must dispatch through it - +# and rejects a linked task worktree, which is the shape bin/fm-spawn.sh always +# hands a crewmate. A crewmate using delegation tools inside its own task +# worktree is legitimate and stays allowed. Any failure to confirm the home is +# inert (exit 0), never a block, so a broken environment never denies a call. +# shellcheck source=bin/fm-primary-scope-lib.sh +. "$SCRIPT_DIR/fm-primary-scope-lib.sh" +fm_primary_scope_matches "$FM_ROOT" "$STATE" || exit 0 + +# Investigation has a dedicated entry point when this home carries it; degrade +# to the two-step brief-then-spawn path when it does not, rather than naming a +# script that is not there. +if [ -f "$FM_ROOT/bin/fm-scout.sh" ]; then + ROUTE='investigation or diagnosis goes to bin/fm-scout.sh "" [project], and ship work goes to bin/fm-brief.sh then bin/fm-spawn.sh' +else + ROUTE='investigation and ship work both go to bin/fm-brief.sh then bin/fm-spawn.sh' +fi + +REASON="[subagent-dispatch] the firstmate primary dispatches through the fleet, not the harness's own delegation tools: work started that way has no durable fleet record, leaves every firstmate guard inert, and dies with this session. Instead, $ROUTE (blocked tool: $TOOL, delegation-shaped on \"$MATCHED\"). Launch the session with FM_ALLOW_SUBAGENT=1 for a deliberate exception." + +json_escape() { + printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' | tr '\n' ' ' +} + +ESCAPED=$(json_escape "$REASON") +printf '{"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"deny"},"systemMessage":"%s"}\n' "$ESCAPED" >&2 +[ "$CLAUDE_MODE" -eq 1 ] || printf '{"decision":"deny","reason":"%s"}\n' "$ESCAPED" +exit 2 diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 2605935..cac6be7 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -125,6 +125,7 @@ family_for_basename() { fm-install-herdr.test.sh|fm-nm-test-contract.test.sh|fm-no-mistakes-ownership.test.sh|\ fm-pi-primary-types.test.sh|\ fm-send-popup-settle.test.sh|fm-send-settle.test.sh|fm-stow-contract.test.sh|\ + fm-subagent-pretool-check.test.sh|\ fm-supervision-instructions.test.sh|fm-tmux-submit-busy.test.sh|fm-transition-lib.test.sh|\ fm-test-run.test.sh|fm-test-isolation-proof.test.sh) printf '%s\n' pure-contract-unit diff --git a/docs/scripts.md b/docs/scripts.md index 92978c2..1b11d23 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -33,6 +33,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-arm-command-policy.mjs` | Semantic owner of the watcher-arm PreToolUse policy (docs/arm-pretool-check.md) | | `fm-continuity-pretool-check.sh` | Narrow Claude recovery gate when in-flight work has no live watcher lock (docs/arm-pretool-check.md) | | `fm-continuity-command-policy.mjs` | Semantic owner of Claude continuity-gate fleet-command classification (docs/arm-pretool-check.md) | +| `fm-subagent-pretool-check.sh` | Primary-home delegation-shape PreToolUse guard (docs/subagent-guard.md) | | `fm-supervision-instructions.sh` | Render the session-start primary-harness supervision block or the one-line repair instruction | | `fm-home-seed.sh` | Transactionally provision a secondmate home and maintain `data/secondmates.md` | | `fm-spawn.sh` | Spawn crewmates, scouts, `id=repo` batches, and secondmates on the resolved harness and runtime backend | diff --git a/docs/subagent-guard.md b/docs/subagent-guard.md new file mode 100644 index 0000000..3212c11 --- /dev/null +++ b/docs/subagent-guard.md @@ -0,0 +1,367 @@ +# Primary-session delegation guard + +This document is the authoritative human-readable contract for the guard that stops a firstmate primary from delegating work outside the fleet. + +The shipped mechanism is `bin/fm-subagent-pretool-check.sh`, a PreToolUse guard that denies a delegation-SHAPED tool name in a genuine primary home. +Claude primaries should also use an untracked per-home local `permissions.deny` list as hardening for known Claude delegation tools, because it removes them from the model's schema entirely. +That deny list must not ship in tracked `.claude/settings.json` because it is Claude-only rather than harness-agnostic, and because tracked project settings propagate into linked worktrees where they disarm legitimate crewmates. + +## Why this exists + +On 2026-07-22 a firstmate primary ran four workers through Claude Code's built-in subagent tool instead of `bin/fm-spawn.sh`. +Three consequences were observed, not hypothesized. + +- The fleet view showed zero work under way for the whole run, because no `state/.meta` and no `data//brief.md` were ever created. +- When the primary session restarted, two of those workers died mid-flight and their work was lost. + A real crewmate lives in its own backend session with durable state and survives a primary restart. +- The supervision cycle then stayed down for 73 minutes unnoticed, which silently killed the captain's Workflowy intake channel, since that channel only fires while a watch cycle runs. + +The deeper defect is that the bypass did not merely skip dispatch, it made the guard stack structurally inert. +Only `bin/fm-spawn.sh` writes `state/.meta`, and every guard keys off that record: `bin/fm-supervision-lib.sh` counts `state/*.meta`, and `bin/fm-turnend-guard.sh` exits silently when that count is zero. +Work started through the harness's own delegation tool writes no metadata, so the in-flight count stayed at zero, the turn-end guard never blocked a blind turn end, and the continuity gate was inert. + +That is the reason the fence has to sit on the harness tool surface, before the primary can create untracked work. +No additional guard keyed on task metadata can catch this class of failure, because the failure is precisely the absence of that metadata. + +## Purpose and boundary + +The guard addresses one concrete, mechanically identifiable event: the primary session reaching for a tool that creates work the fleet will not know about. + +It deliberately does **not** address the broader question of whether a given piece of work should be delegated at all. +That question is a judgment boundary over read-and-think work, it has no tool-shape signal, and a hook that tried to police it would degrade into an advisory nag. +The scope line is therefore: wrong tool reached for, deny; wrong amount of thinking done before reaching for a tool, out of scope. + +The guard is also not a dispatch-quality check. +It says nothing about whether the resulting brief, project, or delivery mode is correct. + +## Shipped mechanism + +`bin/fm-subagent-pretool-check.sh` is the shipped layer. +It classifies the tool NAME by shape rather than against a fixed list. +The tracked Claude PreToolUse matcher is `.*`, so every Claude tool name reaches the script and the script is the single owner of classification. +A stem-enumerating matcher would reintroduce the fail-open-by-enumeration problem this guard exists to solve, because any future tool name outside the matcher would be silently missed before the script could inspect it. +A tool is delegation-shaped when its normalized lowercase name contains one of these stems: + +```text +agent subagent task workflow cron schedul worktree +delegate spawn dispatch handoff remote sendmessage monitor +``` + +Two exclusions keep the shape test from producing false positives. + +- A name beginning `mcp__` is never classified. + An MCP server chooses its own tool names, a task or agent noun there is common, and it has no bearing on fleet dispatch. +- The exact names `taskoutput`, `taskstop`, `taskget`, `tasklist`, `cronlist`, `bashoutput`, and `killshell` are allowed. + These observe or stop work that already exists rather than creating it, and denying them at this layer could strand already-running work with no way to inspect or end it. + A Claude primary's optional local deny list may still remove them from the schema. + The shipped guard stays narrower on purpose so it can never be the reason a runaway task cannot be stopped. + +The shipped guard fires on every delegation-shaped name that reaches it, including future names that no deny list knows about yet. +That future-name behavior is the reason the tracked matcher must match all tools and let the script filter. + +## Recommended Local Claude Deny List + +Claude primaries should add this deny list in untracked per-home local settings, never in tracked `.claude/settings.json`: + +```json +{ + "permissions": { + "deny": [ + "Task", + "Agent", + "Workflow", + "RemoteTrigger", + "Monitor", + "ScheduleWakeup", + "SendMessage", + "EnterWorktree", + "ExitWorktree", + "CronCreate", + "CronDelete", + "CronList", + "TaskCreate", + "TaskGet", + "TaskList", + "TaskUpdate", + "TaskStop", + "TaskOutput" + ] + } +} +``` + +A denied name is removed from the model's schema entirely. +The model is never offered the tool, so there is no call to intercept, no matcher to get wrong, no fail-open path, and no dependence on the model's cooperation. +This is removal, not interception, and it is strictly stronger than any hook. + +This list is recommended local hardening because it closes the known Claude surface before the hook is needed. +It is not tracked for two reasons. + +- It is Claude-only, so it can never be the harness-agnostic shipped fix. +- A tracked `.claude/settings.json` propagates into linked worktrees and disarms legitimate crewmates. + This was verified when a Claude session in a task worktree of this repo lost its `Agent` tool. + +The width of the list remains a captain-owned decision, because denying some of these changes how the captain works with the primary session. +Keep it as one flat local array that is reviewable at a glance and narrowable in one line. +In particular `TaskOutput`, `TaskStop`, `TaskGet`, `TaskList`, and `CronList` only observe or stop work that already exists, but the recommended local deny list still removes them by default. +The hook deliberately allows those names, so the shipped guard can never strand a runaway task with no way to inspect or end it. + +`permissions.allow` is a pre-approval list, not an availability list, so there is no fail-closed positive allowlist available. +That is why any fixed deny list is fail-open against future tools and why the shape-based guard still exists. +The hook cannot re-enable a tool removed from the schema; it only handles a tool name that still reaches PreToolUse. + +### Both `Task` and `Agent` are valid deny keys + +The tool presents to the model as `Agent`. +A prior investigation recorded that the deny key must be `Task` and that using `Agent` "silently does nothing at all". +That is not what this machine shows. + +A five-way A/B with a control, each run in its own directory to rule out settings caching, found that `Task` and `Agent` each independently remove the tool, and that a nonsense name leaves it present. +The full evidence is in the validation record below. + +Pinning both names in the recommended local deny list is correct regardless of which build is running. +It costs one line and removes the failure mode where a rename or a rollback silently reopens the surface. + +## Scope + +The shipped hook fires only in a genuine firstmate primary home, using the shared predicate `fm_primary_scope_matches` from `bin/fm-primary-scope-lib.sh`. +This is the same predicate `bin/fm-sessionstart-nudge.sh` and `bin/fm-turnend-guard.sh` use, so the three tracked primary-scoped hooks cannot drift apart. + +A home is in scope when it has `AGENTS.md`, a `bin/` directory, an existing state directory, and either a plain checkout where git-dir equals git-common-dir or a valid `.fm-secondmate-home` marker. +A marked secondmate home is in scope on purpose: it operates its own fleet and must dispatch through it for the same durability reasons. + +A crewmate's disposable task worktree is a linked git worktree, which is the shape `bin/fm-spawn.sh` always hands out, so it is out of scope. +A crewmate using delegation tools inside its own task worktree is legitimate and stays allowed. +A non-firstmate repo is out of scope. +Any failure to confirm the home is inert, never a block, so a broken environment can never deny a tool call. + +A local Claude deny list is upstream of hook scope and removes known Claude delegation tools wherever Claude applies it. +Do not put that list in tracked project settings, because linked worktrees inherit those settings and would lose legitimate delegation tools. +The hook scope is the shipped enforcement boundary, and the linked-worktree negative case proves the script itself does not block legitimate crewmate delegation. + +## Escape hatch + +`FM_ALLOW_SUBAGENT=1` in the session environment allows the call at the shipped hook. +This is the only escape hatch and the guard fails closed on every other value, including empty, `0`, `yes`, and `true`. + +It is an environment variable rather than a flag, a config file, or a state file because that makes it unforgeable in-session. +The variable must be present when the harness process is launched, so no tool call the agent makes can enable it for the call that follows. +A deliberate use therefore requires restarting the session with the variable set, which is a conscious act, while an accidental use is impossible. + +The escape hatch does not affect any local Claude deny list. +A tool removed from the schema stays removed, so a genuinely intended use of a locally denied tool also requires narrowing or removing that local entry before launch. + +## Output contract + +- Allow returns exit 0 with both streams empty. +- Deny returns exit 2 and writes `{"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"deny"},"systemMessage":"[subagent-dispatch] ..."}` to stderr. +- Default deny mode also writes `{"decision":"deny","reason":"[subagent-dispatch] ..."}` to stdout for Grok. +- `--claude` suppresses stdout completely, because Claude Code ignores a PreToolUse deny when stdout is nonempty. + This is the same verified quirk recorded in [`arm-pretool-check.md`](arm-pretool-check.md), and the tracked Claude hook therefore passes `--claude`. +- Malformed or empty stdin, invalid JSON, a payload with no tool name, and missing `jq` for stdin transport all fail open with exit 0 and no output. + +The deny message names the real dispatch path. +When `bin/fm-scout.sh` exists in the home it routes investigation and diagnosis there and ship work to `bin/fm-brief.sh` then `bin/fm-spawn.sh`. +When that script is absent the message degrades to naming `bin/fm-brief.sh` then `bin/fm-spawn.sh` for both, rather than pointing at a script that is not there. + +## Harness wiring + +Every supported primary harness was reviewed. +Applicability turns on one question: does the harness expose built-in delegation tools that a primary session could use instead of `bin/fm-spawn.sh`? + +| Harness | Delegation surface | Status | +| --- | --- | --- | +| Claude | 18 known tools, listed above | Scoped guard wired and live-verified; untracked local deny list verified and recommended. | +| Codex | none | Not applicable, verified empirically below. Codex 0.144.1 exposes no subagent, sub-task, or delegated-agent tool, so there is nothing to remove or intercept. `.codex/hooks.json` is unchanged. | +| Grok | present, exact tokens unconfirmed | Not wired pending live verification. See below. | +| OpenCode | present, exact tokens unconfirmed | Not wired pending live verification. See below. | +| Pi | none reported | Not wired pending live verification. See below. | + +### Codex, verified not applicable + +Codex 0.144.1 was asked to enumerate its own tools in a scratch git repo on 2026-07-22. + +```sh +codex exec --dangerously-bypass-approvals-and-sandbox --skip-git-repo-check \ + "List the exact names of every tool available to you in this session, one per line, nothing else. Then state on a final line whether you have any tool that spawns a subagent, sub-task, or delegated agent: answer SUBAGENT_TOOL=yes or SUBAGENT_TOOL=no." +``` + +Exact reported tool set and verdict: + +```text +web.run +functions.exec_command +functions.write_stdin +functions.list_mcp_resources +functions.list_mcp_resource_templates +functions.read_mcp_resource +functions.update_plan +functions.request_user_input +functions.request_plugin_install +functions.view_image +functions.get_goal +functions.create_goal +functions.update_goal +functions.apply_patch +image_gen.imagegen +tool_search.tool_search_tool +multi_tool_use.parallel +SUBAGENT_TOOL=no +``` + +`multi_tool_use.parallel` batches calls to the tools above; it does not spawn an agent. +Codex is therefore not applicable today, and this table row is the tripwire: if a future Codex release adds a delegated-agent tool, wire `.codex/hooks.json` the same way its `Bash` PreToolUse entries already forward stdin to a checker. + +### Grok, OpenCode, and Pi, inspected but not wired + +The integration surface of each was inspected and each is structurally wireable for the shipped guard. + +- Grok's tracked hooks (`.grok/hooks/fm-primary-pretool-check.json`, `.grok/hooks/fm-primary-cd-check.json`) use a `PreToolUse` matcher, currently `Bash`, and pipe stdin to a checker. + The checker already reads Grok's `.toolName` field, so only the matcher token is missing. + Grok does expose a delegation surface: `docs/supervision-protocols/grok.md` documents `get_command_or_subagent_output()`, which implies a corresponding dispatch tool. +- OpenCode's tracked plugins gate on `input?.tool !== "bash"` inside `tool.execute.before`, and block by throwing. + Swapping that comparison for a call into this checker with `--tool` is the whole change. +- Pi's tracked extension gates on `event.toolName !== "bash"` inside `pi.on("tool_call", ...)` and blocks by returning `{block: true}`. + The same change applies. A parallel evaluation reports that Pi exposes no delegation tool at all, which would make it not applicable, but that was not verified here. + +None of the three is wired in this change because none of the three binaries is installed on the host where this work was done, so the exact tool-name tokens could not be confirmed and the wiring could not be validated against the real harness. +This repo's rule in the `firstmate-coding-guidelines` skill is that a harness hook must be validated in a scratch project before it is trusted, and `arm-pretool-check.md` records the concrete cost of guessing: a Grok hook whose `command` string is even slightly wrong fails to launch the hook at all. +Wiring an unvalidated matcher would trade a known gap for an unknown breakage. + +The bounded follow-up for each is identical to the Codex procedure above. +On a host with the binary installed, ask the harness to enumerate its tools, then wire the matcher and re-run the live matrix below. +`bin/fm-subagent-pretool-check.sh` needs no change for any of them: it already accepts Grok's stdin shape and the `--tool` CLI form OpenCode and Pi use, and it already emits the Grok stdout decision object by default. + +## Live validation record, 2026-07-22 + +Harness version: + +```text +2.1.217 (Claude Code) +``` + +Every run used a scratch project under this task worktree. +No modified file was installed into the primary checkout or a live harness configuration, and no live watcher, fleet state, or task metadata was used. +The launch command throughout was: + +```sh +claude -p "$PROMPT" --dangerously-skip-permissions --output-format text +``` + +### Tool name and matcher mechanics + +The tool name delivered to PreToolUse hooks was established before any matcher was written, using a throwaway project whose only hook appended `.tool_name` to a log for matcher `.*`. +It logged `Agent` and `Bash`. +A second project using matcher `^(Task|Agent)$` logged `Agent` only, confirming both the live tool name and that Claude Code honors regex anchors in a PreToolUse matcher. +The tracked matcher is now `.*`, matching the throwaway-project evidence above so any future tool name reaches the script classifier. + +### Deny-key A/B, with control + +Prompt: `List the exact names of every tool available to you, comma-separated on one line, nothing else.` +Each variant ran in its own fresh directory to rule out settings caching. + +| `.claude/settings.json` | `Agent` in tool list? | +| --- | --- | +| `{}` | Yes | +| `{"permissions":{"deny":["Task"]}}` | No | +| `{"permissions":{"deny":["Agent"]}}` | No | +| `{"permissions":{"deny":["ZzzNotARealTool"]}}` | Yes | +| `{"permissions":{"deny":["Task","Agent"]}}` | No | + +The nonsense-name control is what makes this conclusive: the tool disappears only when a real name is denied, so the removal is caused by the deny entry rather than by run-to-run variation. +Both `Task` and `Agent` are therefore working deny keys on this build, correcting the earlier claim that only `Task` works. + +The observed baseline surface was 29 tools: + +```text +Agent, Bash, Edit, Read, ReportFindings, ScheduleWakeup, Skill, ToolSearch, Workflow, Write, +CronCreate*, CronDelete*, CronList*, DesignSync*, EnterWorktree*, ExitWorktree*, Monitor*, +NotebookEdit*, PushNotification*, RemoteTrigger*, SendMessage*, TaskCreate*, TaskGet*, +TaskList*, TaskOutput*, TaskStop*, TaskUpdate*, WebFetch*, WebSearch* +``` + +A `*` marks a deferred tool, which is lazy-loaded through `ToolSearch` and does not appear in a plain tool list unless the prompt asks for deferred entries. +This distinction matters when reading the next result: a tool absent from a plain listing is not necessarily denied. + +### Local deny-list hardening + +Run in a scratch firstmate-shaped project containing `AGENTS.md`, `state/`, a full copy of `bin/`, and a Claude settings file containing the recommended local deny-list JSON above. +The result validates the recommended local deny-list JSON above, not tracked repo state. +Asking for deferred entries explicitly returned: + +```text +Bash, Edit, Read, ReportFindings, Skill, ToolSearch, Write, +DesignSync*, NotebookEdit*, PushNotification*, WebFetch*, WebSearch* +``` + +All 18 locally denied names are gone and every ordinary working tool remains, including the five deferred ones. +Comparing against the 29-tool baseline confirms the removal set is exactly the deny list and nothing else. + +### Shipped guard, the case a fixed deny list cannot cover + +To reproduce a future tool that ships before a local deny list is updated, `Workflow` was removed from the deny list in the same scratch project while the guard stayed wired. + +Prompt: `Call the Workflow tool to run any trivial workflow. You must actually attempt the Workflow tool call.` + +Claude reported: + +```text +I attempted the Workflow tool call as requested. It was blocked by a PreToolUse hook in this repo: + +> [subagent-dispatch] the firstmate primary dispatches through the fleet, not the harness's own +> delegation tools... (blocked tool: Workflow). Launch the session with FM_ALLOW_SUBAGENT=1 for a +> deliberate exception. +``` + +This is the load-bearing result: the shipped guard denied a delegation tool that the deny list did not cover, which is the future-name case the shape classifier exists for. + +### Shipped guard scope, the negative case + +The same `Workflow` prompt was then run in a `git worktree add` linked worktree of that scratch project, carrying the identical tracked hook and checker bytes, with no escape hatch. + +```text +The Workflow tool call was not blocked by a hook. It executed normally: launched, ran 1 agent, +and completed successfully returning {"result":"ok"}. +``` + +Same hook, same bytes, deny in the primary home and allow in a crewmate-shaped worktree. +This is the scoping contract working end to end rather than a hook that simply never fires. + +### Escape hatch + +The same `Workflow` prompt in the scratch primary home, launched as `FM_ALLOW_SUBAGENT=1 claude -p ...`: + +```text +Result: the Workflow tool call was NOT blocked by a hook. It launched and ran to completion. +``` + +### Empty-stdout requirement + +A Claude deny is honored only when the hook's stdout is empty. +`tests/fm-subagent-pretool-check.test.sh` asserts stdout is empty on every `--claude` deny and that default mode still emits the Grok object on stdout. +The live consequence is confirmed by the shipped-guard result above: Claude honored the deny and reported the reason text. + +## Automated validation + +`tests/fm-subagent-pretool-check.test.sh` owns the acceptance matrix and is registered in the `pure-contract-unit` family in `bin/fm-test-run.sh`. +It covers the tracked Claude settings boundary that forbids a `permissions` key; the match-all Claude hook registration; denial of every work-creating delegation tool by shape; denial of twelve hypothetical future tool names that appear on no list; the observe-or-stop and MCP exclusions; the scout-present and scout-absent message variants; the escape hatch including its fail-closed values; inertness in a linked task worktree and in a non-firstmate repo; in-scope enforcement for a marked secondmate home; both stdin transports; the empty-stdout requirement; fail-open transport behavior; and the preserved `Bash` seatbelts and `Stop` guard. + +Run: + +```sh +bash -n bin/fm-subagent-pretool-check.sh +bin/fm-lint.sh +tests/fm-subagent-pretool-check.test.sh +``` + +## Known residual gap + +This change does not close the deeper harness-agnostic defect. +Every firstmate guard keys off `state/.meta`, and only `bin/fm-spawn.sh` writes that record. +`bin/fm-supervision-lib.sh` counts `state/*.meta`, and `bin/fm-turnend-guard.sh` exits silently at zero. +Unaccounted primary work therefore reads as idle rather than suspicious. + +The durable fix for that class is to make the guards treat "the primary is doing project-shaped work with zero `state/*.meta` files" as a suspicious state rather than an idle one. +That would catch this class on any harness, including work created through `Bash`. +This change fences only the Claude tool surface. +That is a separate change to `bin/fm-supervision-lib.sh` and `bin/fm-turnend-guard.sh` and is out of scope here. diff --git a/docs/turnend-guard.md b/docs/turnend-guard.md index f4af48a..81143ed 100644 --- a/docs/turnend-guard.md +++ b/docs/turnend-guard.md @@ -4,8 +4,8 @@ This is the authoritative contract for the "no turn ends blind" primary guard re The turn-end supervision predicate lives in `bin/fm-turnend-guard.sh`. Its primary-checkout scope lives in `bin/fm-primary-scope-lib.sh`, shared with the native session-start nudge documented in `docs/sessionstart-nudge.md`. Harness-specific tracked hook files only adapt each verified harness's real turn-end mechanism to that shared predicate. -Two related but separate PreToolUse seatbelts deny a bad command shape before it runs rather than detecting a blind turn end afterward: the watcher-arm seatbelt (`bin/fm-arm-pretool-check.sh`, `docs/arm-pretool-check.md`) and the cd-guard (`bin/fm-cd-pretool-check.sh`, `docs/cd-guard.md`). -Each seatbelt's own document defines its scope; they do not share the turn-end guard's marker-aware primary detection. +Related but separate PreToolUse guards deny a bad tool or command shape before it runs rather than detecting a blind turn end afterward: the watcher-arm seatbelt (`bin/fm-arm-pretool-check.sh`, `docs/arm-pretool-check.md`), the cd-guard (`bin/fm-cd-pretool-check.sh`, `docs/cd-guard.md`), and the primary delegation-shape guard (`bin/fm-subagent-pretool-check.sh`, `docs/subagent-guard.md`). +Each guard's own document defines its scope; do not infer this guard's scoping, loop safety, or fail-open tradeoffs for its PreToolUse siblings. ## Gap Closed diff --git a/tests/fm-subagent-pretool-check.test.sh b/tests/fm-subagent-pretool-check.test.sh new file mode 100755 index 0000000..536cc3d --- /dev/null +++ b/tests/fm-subagent-pretool-check.test.sh @@ -0,0 +1,284 @@ +#!/usr/bin/env bash +# Behavior tests for the primary-session delegation-shape guard: the tracked +# hook registration, shared settings boundary, and PreToolUse classifier. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +CHECK="$ROOT/bin/fm-subagent-pretool-check.sh" +SETTINGS="$ROOT/.claude/settings.json" +TMP_ROOT=$(fm_test_tmproot fm-subagent-pretool-tests) +PRIMARY="$TMP_ROOT/primary" +STATE="$PRIMARY/state" +OUT="$TMP_ROOT/out" +ERR="$TMP_ROOT/err" + +mkdir -p "$PRIMARY/bin" "$STATE" +printf '# fixture\n' > "$PRIMARY/AGENTS.md" +git -C "$PRIMARY" init -q + +BRIEF_ONLY_ROUTE='investigation and ship work both go to bin/fm-brief.sh then bin/fm-spawn.sh' +SCOUT_ROUTE='investigation or diagnosis goes to bin/fm-scout.sh "" [project], and ship work goes to bin/fm-brief.sh then bin/fm-spawn.sh' + +# Every delegation, scheduling, worktree, and task-tracking tool Claude Code +# 2.1.217 offered a primary session in the observed baseline. +# This inventory is shape-classification coverage for the shipped guard and the +# recommended local Claude deny-list hardening list, but tracked settings must +# not ship that Claude-only permissions layer. +DELEGATION_TOOLS='Task Agent Workflow RemoteTrigger Monitor ScheduleWakeup SendMessage EnterWorktree ExitWorktree CronCreate CronDelete CronList TaskCreate TaskGet TaskList TaskUpdate TaskStop TaskOutput' + +# Tools that must stay available: denying these would break ordinary work. +PRESERVED_TOOLS='Bash Edit Read Write Skill ToolSearch WebFetch WebSearch NotebookEdit ReportFindings DesignSync PushNotification' + +run_tool() { + local tool=$1 rc=0 + shift + : > "$OUT" + : > "$ERR" + env FM_ROOT_OVERRIDE="$PRIMARY" FM_HOME="$PRIMARY" FM_STATE_OVERRIDE="$STATE" "$@" \ + "$CHECK" --claude --tool "$tool" > "$OUT" 2> "$ERR" || rc=$? + return "$rc" +} + +expect_allow() { + local label=$1 tool=$2 rc=0 + shift 2 + run_tool "$tool" "$@" || rc=$? + [ "$rc" -eq 0 ] || fail "$label ($tool) must allow, got exit $rc: $(cat "$ERR")" + [ ! -s "$OUT" ] || fail "$label ($tool) allow wrote stdout: $(cat "$OUT")" + [ ! -s "$ERR" ] || fail "$label ($tool) allow wrote stderr: $(cat "$ERR")" +} + +expect_deny() { + local label=$1 tool=$2 rc=0 + run_tool "$tool" || rc=$? + [ "$rc" -eq 2 ] || fail "$label ($tool) must deny with exit 2, got $rc" + [ ! -s "$OUT" ] || fail "$label ($tool) deny wrote stdout: $(cat "$OUT")" + jq -e '.hookSpecificOutput.hookEventName == "PreToolUse" and .hookSpecificOutput.permissionDecision == "deny"' "$ERR" >/dev/null 2>&1 \ + || fail "$label ($tool) deny omitted Claude's permission decision: $(cat "$ERR")" + jq -e --arg tool "$tool" '.systemMessage | startswith("[subagent-dispatch]") and contains("blocked tool: " + $tool)' "$ERR" >/dev/null 2>&1 \ + || fail "$label ($tool) deny message lost its code or tool name: $(jq -r '.systemMessage' "$ERR")" +} + +# --------------------------------------------------------------------------- +# Tracked settings boundary and delegation-shape PreToolUse guard. +# --------------------------------------------------------------------------- + +test_tracked_settings_do_not_ship_permissions_deny() { + jq -e 'keys == ["hooks"] and (has("permissions") | not)' "$SETTINGS" >/dev/null \ + || fail "tracked Claude settings must contain only hooks and no permissions key" + pass "tracked Claude settings do not ship permissions.deny" +} + +test_guard_denies_every_currently_known_delegation_tool() { + local tool + for tool in $DELEGATION_TOOLS; do + case "$tool" in + TaskOutput|TaskStop|TaskGet|TaskList|CronList) continue ;; + esac + expect_deny "known delegation tool" "$tool" + done + pass "the guard independently denies every work-creating delegation tool by shape" +} + +test_guard_denies_hypothetical_future_tools() { + # A fixed deny list is fail-open against tools that do not exist yet. + # None of these names is on any list. + local tool + for tool in SubagentCreate SpawnWorker DelegateTask AgentPool WorkflowRun \ + ScheduleJob CronSchedule CreateWorktree DispatchAgent TaskHandoff \ + RemoteExec BackgroundAgent; do + expect_deny "future delegation tool" "$tool" + done + pass "the guard denies delegation-shaped tools that no deny list knows about yet" +} + +test_guard_allows_ordinary_and_observe_only_tools() { + local tool + for tool in $PRESERVED_TOOLS; do + expect_allow "ordinary tool" "$tool" + done + # Observing or stopping work that already exists is not creating unaccounted + # work, and blocking it would strand a runaway task with no way to end it. + for tool in TaskOutput TaskStop TaskGet TaskList CronList BashOutput KillShell; do + expect_allow "observe-or-stop tool" "$tool" + done + pass "the guard leaves ordinary tools and observe-or-stop operations alone" +} + +test_guard_never_classifies_mcp_tools() { + # An MCP server names its own tools; a task or agent noun there is common and + # has nothing to do with fleet dispatch. + local tool + for tool in mcp__linear__list_issues mcp__tracker__create_task \ + mcp__acme__spawn_agent mcp__slack__slack_send_message; do + expect_allow "MCP tool" "$tool" + done + pass "MCP tool names are never classified as harness delegation" +} + +test_scout_entry_point_named_when_present() { + local actual + printf '#!/usr/bin/env bash\n' > "$PRIMARY/bin/fm-scout.sh" + run_tool Agent && fail "scout-present case must still deny" + actual=$(jq -r '.systemMessage' "$ERR") + case "$actual" in + *"$SCOUT_ROUTE"*) ;; + *) fail "deny must name bin/fm-scout.sh when it exists: $actual" ;; + esac + rm -f "$PRIMARY/bin/fm-scout.sh" + run_tool Agent && fail "scout-absent case must still deny" + actual=$(jq -r '.systemMessage' "$ERR") + case "$actual" in + *"$BRIEF_ONLY_ROUTE"*) ;; + *) fail "deny must degrade to brief-then-spawn when fm-scout.sh is absent: $actual" ;; + esac + pass "deny names bin/fm-scout.sh when it exists and degrades gracefully when it does not" +} + +test_escape_hatch_allows_deliberate_use() { + local rc value + expect_allow "escape hatch set" Agent FM_ALLOW_SUBAGENT=1 + expect_deny "escape hatch unset" Agent + for value in '' 0 yes true 11; do + rc=0 + run_tool Agent "FM_ALLOW_SUBAGENT=$value" || rc=$? + [ "$rc" -eq 2 ] || fail "FM_ALLOW_SUBAGENT='$value' must not release the guard, got exit $rc" + done + pass "the single documented escape hatch releases the guard only on the exact opt-in value" +} + +test_task_worktree_and_non_firstmate_repo_are_inert() { + local child="$TMP_ROOT/child" plain="$TMP_ROOT/plain" rc=0 + git -C "$PRIMARY" config user.name fixture + git -C "$PRIMARY" config user.email fixture@example.test + git -C "$PRIMARY" add AGENTS.md + git -C "$PRIMARY" commit -qm fixture + git -C "$PRIMARY" worktree add -q -b fixture-child "$child" + mkdir -p "$child/bin" "$child/state" + printf '# fixture\n' > "$child/AGENTS.md" + : > "$OUT" + : > "$ERR" + FM_ROOT_OVERRIDE="$child" FM_HOME="$child" FM_STATE_OVERRIDE="$child/state" \ + "$CHECK" --claude --tool Agent > "$OUT" 2> "$ERR" || rc=$? + [ "$rc" -eq 0 ] || fail "a crewmate task worktree must be out of scope, got exit $rc: $(cat "$ERR")" + [ ! -s "$OUT" ] || fail "task-worktree no-op wrote stdout: $(cat "$OUT")" + [ ! -s "$ERR" ] || fail "task-worktree no-op wrote stderr: $(cat "$ERR")" + + mkdir -p "$plain/bin" + git -C "$plain" init -q + rc=0 + FM_ROOT_OVERRIDE="$plain" FM_HOME="$plain" FM_STATE_OVERRIDE="$plain/state" \ + "$CHECK" --claude --tool Agent > "$OUT" 2> "$ERR" || rc=$? + [ "$rc" -eq 0 ] || fail "a non-firstmate repo must be out of scope, got exit $rc" + pass "the guard is inert in a crewmate task worktree and in a non-firstmate repo" +} + +test_secondmate_home_is_in_scope() { + local second="$TMP_ROOT/second" rc=0 + git -C "$PRIMARY" worktree add -q -b fixture-second "$second" + mkdir -p "$second/bin" "$second/state" + printf '# fixture\n' > "$second/AGENTS.md" + printf 'sm-fixture\n' > "$second/.fm-secondmate-home" + FM_ROOT_OVERRIDE="$second" FM_HOME="$second" FM_STATE_OVERRIDE="$second/state" \ + "$CHECK" --claude --tool Agent > "$OUT" 2> "$ERR" || rc=$? + [ "$rc" -eq 2 ] || fail "a marked secondmate home operates a fleet and must be guarded, got exit $rc" + pass "a marked secondmate home is guarded even though it is a linked worktree" +} + +test_stdin_transports_and_output_shapes() { + local rc=0 + : > "$OUT"; : > "$ERR" + printf '%s' '{"tool_name":"Agent","tool_input":{"prompt":"go"}}' \ + | FM_ROOT_OVERRIDE="$PRIMARY" FM_HOME="$PRIMARY" FM_STATE_OVERRIDE="$STATE" \ + "$CHECK" --claude > "$OUT" 2> "$ERR" || rc=$? + [ "$rc" -eq 2 ] || fail "Claude-shaped stdin must deny, got exit $rc" + [ ! -s "$OUT" ] || fail "Claude deny wrote stdout, which makes Claude ignore the deny: $(cat "$OUT")" + + rc=0 + : > "$OUT"; : > "$ERR" + printf '%s' '{"toolName":"Agent"}' \ + | FM_ROOT_OVERRIDE="$PRIMARY" FM_HOME="$PRIMARY" FM_STATE_OVERRIDE="$STATE" \ + "$CHECK" > "$OUT" 2> "$ERR" || rc=$? + [ "$rc" -eq 2 ] || fail "Grok-shaped stdin must deny, got exit $rc" + jq -e '.decision == "deny" and (.reason | startswith("[subagent-dispatch]"))' "$OUT" >/dev/null 2>&1 \ + || fail "default deny mode must write a Grok decision object on stdout: $(cat "$OUT")" + + rc=0 + : > "$OUT"; : > "$ERR" + printf '%s' '{"tool_name":"Bash","tool_input":{"command":"ls"}}' \ + | FM_ROOT_OVERRIDE="$PRIMARY" FM_HOME="$PRIMARY" FM_STATE_OVERRIDE="$STATE" \ + "$CHECK" --claude > "$OUT" 2> "$ERR" || rc=$? + [ "$rc" -eq 0 ] || fail "Bash through stdin must allow, got exit $rc" + [ ! -s "$OUT" ] && [ ! -s "$ERR" ] || fail "stdin allow wrote output" + pass "both stdin transports classify correctly and Claude's deny keeps stdout empty" +} + +test_malformed_transport_fails_open() { + local rc payload + for payload in '{not-json' '' '{}' '{"tool_name":null}'; do + rc=0 + : > "$OUT"; : > "$ERR" + printf '%s' "$payload" \ + | FM_ROOT_OVERRIDE="$PRIMARY" FM_HOME="$PRIMARY" FM_STATE_OVERRIDE="$STATE" \ + "$CHECK" --claude > "$OUT" 2> "$ERR" || rc=$? + [ "$rc" -eq 0 ] || fail "malformed transport must fail open, payload '$payload' gave exit $rc" + [ ! -s "$OUT" ] || fail "fail-open path wrote stdout for payload '$payload'" + done + pass "malformed, empty, and tool-name-less payloads fail open rather than blocking every tool call" +} + +test_missing_jq_stdin_transport_fails_open() { + local fakebin="$TMP_ROOT/no-jq-bin" bash_bin cat_bin rc=0 + bash_bin=$(command -v bash) || fail "test needs bash to simulate the hook shebang" + cat_bin=$(command -v cat) || fail "test needs cat to feed stdin without jq" + mkdir -p "$fakebin" + ln -sf "$bash_bin" "$fakebin/bash" + ln -sf "$cat_bin" "$fakebin/cat" + : > "$OUT"; : > "$ERR" + printf '%s' '{"tool_name":"Agent"}' \ + | env PATH="$fakebin" FM_ROOT_OVERRIDE="$PRIMARY" FM_HOME="$PRIMARY" FM_STATE_OVERRIDE="$STATE" \ + "$CHECK" --claude > "$OUT" 2> "$ERR" || rc=$? + [ "$rc" -eq 0 ] || fail "missing jq transport must fail open, got exit $rc: $(cat "$ERR")" + [ ! -s "$OUT" ] || fail "missing jq fail-open path wrote stdout: $(cat "$OUT")" + [ ! -s "$ERR" ] || fail "missing jq fail-open path wrote stderr: $(cat "$ERR")" + pass "missing jq for stdin transport fails open rather than denying every tool call" +} + +test_claude_hook_registration_preserves_bash_seatbelts() { + jq -e ' + [.hooks.PreToolUse[] | .hooks[].command] + | any(contains("fm-subagent-pretool-check.sh --claude")) + ' "$SETTINGS" >/dev/null || fail "Claude settings omit the delegation-shape PreToolUse guard" + # A stem-enumerating matcher repeats the fail-open-by-enumeration defect the + # script exists to remove. Match all tools and let the script be the single + # owner of classification. + jq -e ' + [.hooks.PreToolUse[] | select(.hooks[].command | contains("fm-subagent-pretool-check.sh")) | .matcher] | .[0] + | . == ".*" + ' "$SETTINGS" >/dev/null || fail "the guard matcher must match all tools" + jq -e ' + [.hooks.PreToolUse[] | select(.matcher == "Bash") | .hooks[].command] as $bash + | ($bash | any(contains("fm-arm-pretool-check.sh"))) + and ($bash | any(contains("fm-cd-pretool-check.sh"))) + and ($bash | any(contains("fm-continuity-pretool-check.sh"))) + ' "$SETTINGS" >/dev/null || fail "the existing Bash PreToolUse seatbelts changed" + jq -e '.hooks.Stop[0].hooks[0].command | contains("fm-turnend-guard.sh")' "$SETTINGS" >/dev/null \ + || fail "the Stop turn-end guard changed" + pass "Claude wires the guard while preserving the Bash seatbelts and the Stop guard" +} + +test_tracked_settings_do_not_ship_permissions_deny +test_guard_denies_every_currently_known_delegation_tool +test_guard_denies_hypothetical_future_tools +test_guard_allows_ordinary_and_observe_only_tools +test_guard_never_classifies_mcp_tools +test_scout_entry_point_named_when_present +test_escape_hatch_allows_deliberate_use +test_task_worktree_and_non_firstmate_repo_are_inert +test_secondmate_home_is_in_scope +test_stdin_transports_and_output_shapes +test_malformed_transport_fails_open +test_missing_jq_stdin_transport_fails_open +test_claude_hook_registration_preserves_bash_seatbelts From f5bdea30075158139b2080ec0271a9e04b0975f2 Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:46:26 -0700 Subject: [PATCH 02/34] fix: install tasks-axi in portable CI shards (#866) Reproduction: portable-parallel-2 completed successfully without tasks-axi while fm-decision-hold-lifecycle emitted a gate skip in 30 ms. The pre-shard lane installed tasks-axi and exercised the test fully. Installing tasks-axi is the smallest counterfactual and makes the representative shard execute the test with gate_skip=false in about 20 seconds. Both parallel jobs receive symmetric setup, while the exact 91-test inventory and coverage guard remain unchanged. --- .github/workflows/ci.yml | 10 ++++++++++ tests/fm-test-run.test.sh | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3c4fb5..62dabea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,11 @@ jobs: set -eu bin/fm-install-shellcheck.sh "$RUNNER_TEMP/bin" echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + - name: Install tasks-axi + run: | + set -eu + npm install -g tasks-axi + tasks-axi --version - name: Run portable parallel shard 1 run: | set -eu @@ -78,6 +83,11 @@ jobs: set -eu bin/fm-install-shellcheck.sh "$RUNNER_TEMP/bin" echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + - name: Install tasks-axi + run: | + set -eu + npm install -g tasks-axi + tasks-axi --version - name: Run portable parallel shard 2 run: | set -eu diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh index 976f01a..e0714cd 100755 --- a/tests/fm-test-run.test.sh +++ b/tests/fm-test-run.test.sh @@ -366,6 +366,18 @@ test_ci_and_docs_call_the_owner() { || fail "CI shard 1 must invoke --lane portable-parallel-1" grep -Fq 'bin/fm-test-run.sh --lane portable-parallel-2' "$CI" \ || fail "CI shard 2 must invoke --lane portable-parallel-2" + local shard job_body + for shard in 1 2; do + job_body=$(awk -v job=" tests-portable-parallel-$shard:" ' + $0 == job { in_job=1; next } + in_job && /^ [a-zA-Z0-9_-]+:/ { exit } + in_job { print } + ' "$CI") + printf '%s\n' "$job_body" | grep -Fq 'npm install -g tasks-axi' \ + || fail "CI portable parallel shard $shard must install tasks-axi" + printf '%s\n' "$job_body" | grep -Fq 'tasks-axi --version' \ + || fail "CI portable parallel shard $shard must verify tasks-axi" + done grep -Fq 'bin/fm-test-run.sh --lane portable-serial' "$CI" \ || fail "CI portable serial must invoke --lane portable-serial" grep -Fq 'bin/fm-test-run.sh --check-coverage' "$CI" \ From 51404137e8c4729670233cc31ff43eeae527b77c Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:57:58 -0700 Subject: [PATCH 03/34] feat(bin): make dispatch profiles quota aware (#867) * feat: make dispatch profiles quota aware * no-mistakes(review): Fix quota window and Grok product scoping * no-mistakes(document): Document implicit quota-aware dispatch accurately --- .agents/skills/bootstrap-diagnostics/SKILL.md | 2 +- bin/fm-bootstrap.sh | 49 ++- bin/fm-dispatch-select.sh | 281 ++++++++----- docs/architecture.md | 4 +- docs/configuration.md | 27 +- docs/examples/crew-dispatch.json | 8 +- docs/scripts.md | 2 +- tests/fm-bootstrap.test.sh | 12 +- tests/fm-dispatch-select.test.sh | 371 ++++++++++++------ tests/fm-spawn-dispatch-profile.test.sh | 35 ++ 10 files changed, 547 insertions(+), 244 deletions(-) diff --git a/.agents/skills/bootstrap-diagnostics/SKILL.md b/.agents/skills/bootstrap-diagnostics/SKILL.md index 102071b..9d9ffe1 100644 --- a/.agents/skills/bootstrap-diagnostics/SKILL.md +++ b/.agents/skills/bootstrap-diagnostics/SKILL.md @@ -20,7 +20,7 @@ When any diagnostic needs captain attention, report the plain consequence and re For `treehouse`, this also covers an installed version whose `treehouse get` lacks `--lease`; treat it as an upgrade request. For `no-mistakes`, this also covers an installed version older than 1.31.2, because crewmate validation briefs delegate gate mechanics to no-mistakes' version-matched guidance. For `tasks-axi`, this also covers an installed build that fails the compatibility probe (`docs/configuration.md` "Backlog backend" owns the definition); `config/backlog-backend=manual` only suppresses the verbose `BOOTSTRAP_INFO: tasks-axi available` fact, not this missing-tool report. - For `quota-axi`, bootstrap requires it because crew-dispatch `quota-balanced` may call it; `bin/fm-dispatch-select.sh` still degrades at runtime when quota data is unavailable. + For `quota-axi`, bootstrap requires it because every crew-dispatch profile array calls it automatically; `bin/fm-dispatch-select.sh` still selects uniformly from valid candidates with OS-backed randomness when quota data is unavailable. - `MISSING_MANUAL: (instructions: )` - tell the captain why the tool is required and give them the printed instructions URL, but do not pass the tool to `bin/fm-bootstrap.sh install`; wait for the captain to complete the manual installation, then rerun session start to confirm the dependency is present. - `BACKEND_INVALID: (known: )` - the resolved runtime backend has no verified dependency or lifecycle contract, so do not dispatch work until the invalid `FM_BACKEND` or `config/backend` value is corrected to one of the listed backends. - `NEEDS_GH_AUTH` - ask the captain to run `! gh auth login` (interactive; you cannot run it for them). diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index 96ad035..c22422a 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -52,8 +52,9 @@ # with update --archive-body and mv [...]); an installed but # incompatible build reports MISSING like no-mistakes. A compatible # tasks-axi default backend is silent. quota-axi is required because -# crew-dispatch quota-balanced may call it; fm-dispatch-select.sh still -# degrades at runtime when quota data is unavailable. +# every crew-dispatch profile array calls it automatically; +# fm-dispatch-select.sh still uses OS-backed random selection across +# valid candidates when quota data is unavailable. # X mode is OPTIONAL and inert unless FM_HOME/.env has a non-empty # FMX_PAIRING_TOKEN. When opted in, bootstrap requires curl+jq, writes # the relay poll shim and 30s cadence config, and prints an FMX line. @@ -722,14 +723,20 @@ crew_dispatch_validate() { elif $h == "opencode" then false else true end; - def use_profiles($u): - if ($u | type) == "array" then $u - elif ($u | type) == "object" then [$u] + def profiles($value): + if ($value | type) == "array" then $value + elif ($value | type) == "object" then [$value] else [] end; + def configured_profiles: + ([(.rules // [])[]? | profiles(.use?)[]?] + + (if has("default") then [profiles(.default)[]?] else [] end)); + def malformed_optional_fields($items): + ($items | any(has("model") and (((.model | type) != "string") or (.model | length) == 0))) + or ($items | any(has("effort") and (((.effort | type) != "string") or (.effort | length) == 0))); def bad_efforts: - ([(.rules // [])[]? | use_profiles(.use?)[]? | {h: .harness, e: .effort}] - + (if (.default? | type) == "object" then [{h: .default.harness, e: .default.effort}] else [] end)) + configured_profiles + | map({h: .harness, e: .effort}) | map(select(.e != null)) | map(select((.h | type) == "string" and verified(.h))) | map(select(. as $p | effort_ok($p.h; $p.e) | not)) @@ -741,15 +748,20 @@ crew_dispatch_validate() { elif [(.rules // [])[]? | select((.when? | type) != "string" or (.when | length) == 0)] | length > 0 then "each rule needs non-empty when" elif [(.rules // [])[]? | select((.use? | type) != "object" and (.use? | type) != "array")] | length > 0 then "each rule needs use" elif [(.rules // [])[]? | select((.use? | type) == "array" and (.use | length) == 0)] | length > 0 then "each rule needs at least one use profile" - elif [(.rules // [])[]? | use_profiles(.use?)[]? | select(type != "object")] | length > 0 then "each use profile must be an object" - elif [(.rules // [])[]? | use_profiles(.use?)[]? | select((.harness? | type) != "string" or (.harness | length) == 0)] | length > 0 then "each use profile needs harness" + elif [(.rules // [])[]? | profiles(.use?)[]? | select(type != "object")] | length > 0 then "each use profile must be an object" + elif [(.rules // [])[]? | profiles(.use?)[]? | select((.harness? | type) != "string" or (.harness | length) == 0)] | length > 0 then "each use profile needs harness" + elif malformed_optional_fields([(.rules // [])[]? | profiles(.use?)[]?]) then "use profile model and effort must be non-empty strings when present" elif [(.rules // [])[]? | select(has("select") and ((.select? | type) != "string" or (.select | length) == 0))] | length > 0 then "select must be a non-empty string" elif [(.rules // [])[]? | .select? // empty | select(. != "quota-balanced")] | length > 0 then "unknown select: " + ([ (.rules // [])[]? | .select? // empty | select(. != "quota-balanced") ] | unique | join(", ")) - elif has("default") and (.default | type) != "object" then "default must be an object" - elif has("default") and ((.default.harness? | type) != "string" or (.default.harness | length) == 0) then "default needs harness when present" + elif has("default") and ((.default | type) != "object" and (.default | type) != "array") then "default must be a profile object or non-empty profile array" + elif has("default") and ((.default | type) == "array" and (.default | length) == 0) then "default needs at least one profile" + elif has("default") and ([profiles(.default)[]? | select(type != "object")] | length) > 0 then "each default profile must be an object" + elif has("default") and ([profiles(.default)[]? | select((.harness? | type) != "string" or (.harness | length) == 0)] | length) > 0 then "each default profile needs harness" + elif has("default") and malformed_optional_fields([profiles(.default)[]?]) then "default profile model and effort must be non-empty strings when present" else - ([(.rules // [])[]? | use_profiles(.use?)[]?.harness] + [.default?.harness?] + (configured_profiles + | map(.harness) | map(select(. != null)) | map(select(. as $h | verified($h) | not)) | unique) as $bad_harnesses @@ -771,15 +783,14 @@ crew_dispatch_validate() { elif ($p.effort? != null) then "/default" else "" end) + (if ($p.effort? != null) then "/" + ($p.effort | tostring) else "" end); - def use_label($r): - if ($r.use | type) == "array" then - ((if ($r.select? != null) then ($r.select | tostring) else "first" end) - + "[" + ([$r.use[] | profile(.)] | join(", ")) + "]") - else profile($r.use) + def profile_set($value; $selector): + if ($value | type) == "array" then + (($selector // "quota-balanced") + "[" + ([$value[] | profile(.)] | join(", ")) + "]") + else profile($value) end; (["BOOTSTRAP_INFO: crew dispatch active config/crew-dispatch.json"] - + [(.rules // [])[]? | "BOOTSTRAP_INFO: crew dispatch rule: " + (.when | tostring) + " -> " + use_label(.)] - + (if (.default? | type) == "object" then ["BOOTSTRAP_INFO: crew dispatch default: " + profile(.default)] else [] end)) + + [(.rules // [])[]? | "BOOTSTRAP_INFO: crew dispatch rule: " + (.when | tostring) + " -> " + profile_set(.use; .select?)] + + (if has("default") then ["BOOTSTRAP_INFO: crew dispatch default: " + profile_set(.default; null)] else [] end)) | .[] ' "$file" fi diff --git a/bin/fm-dispatch-select.sh b/bin/fm-dispatch-select.sh index c626e9a..5caf59f 100755 --- a/bin/fm-dispatch-select.sh +++ b/bin/fm-dispatch-select.sh @@ -1,34 +1,39 @@ #!/usr/bin/env bash -# Resolve one already-matched crew-dispatch rule to a concrete profile. +# Resolve one already-matched crew-dispatch rule or default to a concrete profile. # Usage: -# fm-dispatch-select.sh [--select ] [--quota-json ] [] +# fm-dispatch-select.sh [--select ] [--quota-json ] [] # # Input may be a full rule object with `use` and optional `select`, a single -# profile object, or an ordered array of profile objects. +# profile object, or a non-empty array of profile objects. # Output is one compact JSON profile object on stdout. +# Selection diagnostics go to stderr and never alter the profile JSON. # -# quota-balanced is deterministic, and this header is the single owner of its -# contract: -# - It runs quota-axi --json (or the --quota-json fixture). -# - Per candidate vendor it takes the minimum percentRemaining across that -# vendor's GENERAL windows only - Claude five_hour and seven_day, Codex -# five_hour and weekly - ignoring model-scoped windows such as model:fable -# and model:codex_bengalfox:*. -# - The vendor with the higher minimum remaining quota wins; an exact tie -# between equally trusted candidates uses the first array element. -# - Stale-but-cached general-window numbers are usable, but a fresh candidate -# wins unless the stale candidate's minimum is at least the stale-clear -# margin higher (default 20 points - the definition of "clearly less -# constrained"). -# - A vendor absent from quota output, or with no usable general windows, is -# unavailable; selection happens among available candidates. -# - If quota-axi is missing, exits non-zero, returns unparseable JSON, or no -# candidate is usable, the reason is logged to stderr and the first array -# element is printed - quota trouble never blocks dispatch. +# This header is the single owner of quota-aware selection mechanics: +# - A profile object resolves to itself for backward compatibility. +# - Every profile array is quota-aware, whether or not it carries the legacy +# explicit `select: "quota-balanced"` strategy. +# - It runs the installed quota-axi --json (or the --quota-json fixture). +# - Candidates map to the quota provider and product their model consumes: +# direct Claude -> Claude, direct Codex -> Codex, direct Grok -> Grok Build, +# and Pi/OpenCode models prefixed anthropic/, openai-codex/, or xai/ -> +# Claude, Codex, or the xAI API product respectively. +# - A candidate's score is the minimum percentRemaining among its relevant +# general and matching model windows, or its exact Grok product window. +# Grok's aggregate credits window is used only when product windows are not +# exposed, so Grok Build and xAI API remain distinct. +# - Unscorable candidates never beat candidates with usable quota data. +# - Stale-but-cached numbers remain usable, but a fresh candidate wins unless +# the best stale score is at least the stale-clear margin higher (default +# 20 points). Equal winning scores use a random tie-break. +# - If quota-axi is unavailable, fails, returns unusable data, or no candidate +# can be scored, selection falls back uniformly across every valid candidate +# using rejection sampling over a 32-bit value from /dev/urandom. +# - Runtime quota trouble never turns malformed profile JSON into a fallback; +# invalid input exits 2 with an actionable validation error. # -# quota-balanced uses quota-axi --json unless --quota-json supplies a fixture. # FM_DISPATCH_QUOTA_AXI overrides the quota command. # FM_DISPATCH_STALE_CLEAR_MARGIN overrides the default 20 point stale margin. +# FM_DISPATCH_RANDOM_SOURCE overrides /dev/urandom for deterministic tests only. set -u STALE_CLEAR_MARGIN=${FM_DISPATCH_STALE_CLEAR_MARGIN:-20} @@ -92,6 +97,7 @@ done [ "${#ARGS[@]}" -le 1 ] || { echo "error: expected at most one JSON argument" >&2; exit 2; } command -v jq >/dev/null 2>&1 || { echo "error: jq is required" >&2; exit 2; } +command -v od >/dev/null 2>&1 || { echo "error: od is required for OS-backed random selection" >&2; exit 2; } if [ "${#ARGS[@]}" -eq 1 ]; then SPEC_JSON=${ARGS[0]} @@ -103,132 +109,229 @@ profiles_json=$(printf '%s\n' "$SPEC_JSON" | jq -ec ' (if type == "object" and has("use") then .use else . end) | if type == "array" then . elif type == "object" then [.] - else empty + else error("dispatch input must be a rule, profile, or profile array") end ' 2>/dev/null) || { echo "error: dispatch input must be a rule, profile, or profile array" >&2; exit 2; } -profile_count=$(printf '%s\n' "$profiles_json" | jq 'length') -[ "$profile_count" -gt 0 ] || { echo "error: dispatch profile array must not be empty" >&2; exit 2; } +validation_error=$(printf '%s\n' "$profiles_json" | jq -r ' + def verified($h): ["claude", "codex", "opencode", "pi", "grok"] | index($h); + def effort_ok($h; $e): + if $h == "claude" then ["low", "medium", "high", "xhigh", "max"] | index($e) + elif $h == "codex" then ["low", "medium", "high", "xhigh"] | index($e) + elif $h == "grok" then ["low", "medium", "high"] | index($e) + elif $h == "pi" then ["low", "medium", "high", "xhigh", "max"] | index($e) + elif $h == "opencode" then false + else false + end; + if length == 0 then "dispatch profile array must not be empty" + elif any(.[]; type != "object") then "each dispatch profile must be an object" + elif any(.[]; ((.harness? | type) != "string") or (.harness | length) == 0) then "each dispatch profile needs a non-empty harness" + elif any(.[]; has("model") and (((.model | type) != "string") or (.model | length) == 0)) then "dispatch profile model must be a non-empty string when present" + elif any(.[]; has("effort") and (((.effort | type) != "string") or (.effort | length) == 0)) then "dispatch profile effort must be a non-empty string when present" + elif any(.[]; .harness as $h | verified($h) | not) then "dispatch profile contains an unverified harness" + elif any(.[]; has("effort") and (. as $profile | effort_ok($profile.harness; $profile.effort) | not)) then "dispatch profile contains an unsupported harness/effort pair" + else empty + end +') +[ -z "$validation_error" ] || { echo "error: $validation_error" >&2; exit 2; } -first_profile() { - printf '%s\n' "$profiles_json" | jq -c ' +clean_profile_at() { + local index=$1 + printf '%s\n' "$profiles_json" | jq -c --argjson index "$index" ' def clean($p): {harness: $p.harness} + (if ($p.model? | type) == "string" then {model: $p.model} else {} end) + (if ($p.effort? | type) == "string" then {effort: $p.effort} else {} end); - clean(.[0]) + clean(.[$index]) ' } +random_index() { + local count=$1 source raw ceiling attempts + source=${FM_DISPATCH_RANDOM_SOURCE:-/dev/urandom} + [ "$count" -gt 0 ] || return 1 + [ -r "$source" ] || return 1 + ceiling=$((4294967296 - (4294967296 % count))) + attempts=0 + while [ "$attempts" -lt 32 ]; do + raw=$(LC_ALL=C od -An -N4 -tu4 "$source" 2>/dev/null | tr -d '[:space:]') + case "$raw" in + ''|*[!0-9]*) attempts=$((attempts + 1)); continue ;; + esac + if [ "$raw" -lt "$ceiling" ]; then + printf '%s\n' "$((raw % count))" + return 0 + fi + attempts=$((attempts + 1)) + done + return 1 +} + +random_profile() { + local reason count index + reason=$1 + count=$(printf '%s\n' "$profiles_json" | jq 'length') + if ! index=$(random_index "$count"); then + echo "error: OS-backed random source is unavailable" >&2 + exit 1 + fi + log "$reason" + log "selection basis: random fallback" + clean_profile_at "$index" +} + select_strategy=$SELECT_OVERRIDE if [ -z "$select_strategy" ]; then select_strategy=$(printf '%s\n' "$SPEC_JSON" | jq -r ' if type == "object" and has("use") and (.select? | type) == "string" then .select else "" end ' 2>/dev/null || true) fi +if [ -n "$select_strategy" ] && [ "$select_strategy" != quota-balanced ]; then + echo "error: unknown select strategy '$select_strategy'" >&2 + exit 2 +fi -if [ "$select_strategy" != quota-balanced ]; then - if [ -n "$select_strategy" ]; then - log "unknown select strategy '$select_strategy'; using first profile" - fi - first_profile +is_array=$(printf '%s\n' "$SPEC_JSON" | jq -r ' + if type == "object" and has("use") then (.use | type) == "array" else type == "array" end +') +if [ "$is_array" != true ] && [ -z "$select_strategy" ]; then + log "selection basis: single profile" + clean_profile_at 0 exit 0 fi if [ -n "$QUOTA_JSON_FILE" ]; then if ! quota_json=$(cat "$QUOTA_JSON_FILE" 2>/dev/null); then - log "cannot read quota JSON; using first profile" - first_profile + random_profile "cannot read quota JSON" exit 0 fi else quota_cmd=${FM_DISPATCH_QUOTA_AXI:-quota-axi} if ! command -v "$quota_cmd" >/dev/null 2>&1; then - log "quota-axi missing; using first profile" - first_profile + random_profile "quota-axi missing" exit 0 fi quota_json=$("$quota_cmd" --json 2>/dev/null) quota_status=$? if [ "$quota_status" -ne 0 ]; then - log "quota-axi exited $quota_status; using first profile" - first_profile + random_profile "quota-axi exited $quota_status" exit 0 fi fi if ! printf '%s\n' "$quota_json" | jq -e 'type == "object" and (.providers | type) == "array"' >/dev/null 2>&1; then - log "quota-axi returned unparseable JSON; using first profile" - first_profile + random_profile "quota-axi returned unparseable JSON" exit 0 fi selection=$(printf '%s\n' "$quota_json" | jq -ec \ --argjson profiles "$profiles_json" \ --argjson margin "$STALE_CLEAR_MARGIN" ' - def clean($p): - {harness: $p.harness} - + (if ($p.model? | type) == "string" then {model: $p.model} else {} end) - + (if ($p.effort? | type) == "string" then {effort: $p.effort} else {} end); - def provider_for($h): [.providers[]? | select(.provider == $h)][0]; - def general_ids($h): - if $h == "claude" then ["five_hour", "seven_day"] - elif $h == "codex" then ["five_hour", "weekly"] - else [] + def clean_text: + ascii_downcase | gsub("[^a-z0-9]"; ""); + def model_name($model): + ($model | split("/") | last | split(":") | first); + def route($profile): + ($profile.harness // "") as $h + | ($profile.model // "") as $model + | if $h == "claude" then {provider: "claude", model: $model} + elif $h == "codex" then {provider: "codex", model: $model} + elif $h == "grok" then {provider: "grok", product: "grok_build", model: $model} + elif (($h == "pi" or $h == "opencode") and ($model | startswith("anthropic/"))) then + {provider: "claude", model: (model_name($model))} + elif (($h == "pi" or $h == "opencode") and ($model | startswith("openai-codex/"))) then + {provider: "codex", model: (model_name($model))} + elif (($h == "pi" or $h == "opencode") and ($model | startswith("xai/"))) then + {provider: "grok", product: "api", model: (model_name($model))} + else null + end; + def provider_for($id): [.providers[]? | select(.provider == $id)][0]; + def model_window_matches($window; $model): + if (($window.kind? // "") != "model") or ($model | length) == 0 then false + else + (($window.id? // "") + " " + ($window.label? // "") | clean_text) as $scope + | ($model | clean_text) as $wanted + | (($scope | contains($wanted)) or ($wanted | contains($scope)) + or (["fable", "opus", "haiku", "sonnet", "spark"] + | map(. as $family | ($scope | contains($family)) and ($wanted | contains($family))) + | any)) end; - def candidate_metric($p; $i): - . as $root - | ($p.harness // "") as $h - | ($root | provider_for($h)) as $provider - | if ($provider == null) or ((general_ids($h) | length) == 0) then empty + def usable_percent($window): + (($window.percentRemaining? | type) == "number") + and ($window.percentRemaining >= 0) + and ($window.percentRemaining <= 100); + def general_window_matches($window; $provider): + if $provider == "claude" then ["five_hour", "seven_day"] | index($window.id? // "") != null + elif $provider == "codex" then ["five_hour", "weekly"] | index($window.id? // "") != null + else false + end; + def relevant_windows($provider; $route): + ($provider.windows // []) as $all_windows + | ($all_windows | map(select(usable_percent(.)))) as $windows + | if $route.provider == "grok" then + ($windows | map(select(.id == ("product:" + $route.product)))) as $product_windows + | if ($product_windows | length) > 0 then $product_windows + elif ($all_windows | map(select((.id? // "") | startswith("product:"))) | length) == 0 then + ($windows | map(select(.id == "credits"))) + else [] + end else - (($provider.windows // []) - | map(. as $window - | select(((general_ids($h) | index($window.id)) != null) - and (($window.kind? // "") != "model") - and (($window.percentRemaining? | type) == "number")))) as $windows + $windows | map(select( + general_window_matches(.; $route.provider) + or model_window_matches(.; $route.model) + )) + end; + def candidate_metric($profile; $index): + . as $root + | route($profile) as $route + | if $route == null then empty + else ($root | provider_for($route.provider)) as $provider + | if ($provider == null) or (["fresh", "stale"] | index($provider.state.status? // "") | not) then empty + else relevant_windows($provider; $route) as $windows | if ($windows | length) == 0 then empty else { - index: $i, - profile: clean($p), - harness: $h, - min: ($windows | map(.percentRemaining) | min), + index: $index, + score: ($windows | map(.percentRemaining) | min), fresh: (($provider.state.status? // "") == "fresh") } end + end end; - def better($a; $b): - if $a == null then $b - elif $b == null then $a - elif ($b.min > $a.min) then $b - elif ($b.min == $a.min and $b.index < $a.index) then $b - else $a - end; - def best_by_min($xs): reduce $xs[] as $x (null; better(.; $x)); + def best_score($items): if ($items | length) == 0 then null else ($items | map(.score) | max) end; . as $quota_root - | ([$profiles | to_entries[] | . as $entry | ($quota_root | candidate_metric($entry.value; $entry.key))]) as $candidates - | if ($candidates | length) == 0 then { - fallback: true, - reason: "no usable quota windows for candidate vendors", - profile: clean($profiles[0]) - } + | ([$profiles | to_entries[] | . as $entry + | ($quota_root | candidate_metric($entry.value; $entry.key))]) as $candidates + | if ($candidates | length) == 0 then {fallback: true} else - (best_by_min($candidates | map(select(.fresh)))) as $fresh_best - | (best_by_min($candidates | map(select(.fresh | not)))) as $stale_best + ($candidates | map(select(.fresh))) as $fresh + | ($candidates | map(select(.fresh | not))) as $stale + | best_score($fresh) as $fresh_best + | best_score($stale) as $stale_best | (if $fresh_best != null and $stale_best != null then - if $stale_best.min >= ($fresh_best.min + $margin) then $stale_best else $fresh_best end - elif $fresh_best != null then $fresh_best - else $stale_best - end) as $chosen - | {fallback: false, profile: $chosen.profile} + if $stale_best >= ($fresh_best + $margin) then {items: $stale, score: $stale_best} + else {items: $fresh, score: $fresh_best} + end + elif $fresh_best != null then {items: $fresh, score: $fresh_best} + else {items: $stale, score: $stale_best} + end) as $winning + | {fallback: false, indices: [$winning.items[] | select(.score == $winning.score) | .index]} end ' 2>/dev/null) || { - log "quota-axi data could not be evaluated; using first profile" - first_profile + random_profile "quota-axi data could not be evaluated" exit 0 } if [ "$(printf '%s\n' "$selection" | jq -r '.fallback')" = true ]; then - log "$(printf '%s\n' "$selection" | jq -r '.reason'); using first profile" + random_profile "no usable quota windows for candidates" + exit 0 +fi + +winner_indices=$(printf '%s\n' "$selection" | jq -c '.indices') +winner_count=$(printf '%s\n' "$winner_indices" | jq 'length') +if ! winner_offset=$(random_index "$winner_count"); then + echo "error: OS-backed random source is unavailable" >&2 + exit 1 fi -printf '%s\n' "$selection" | jq -c '.profile' +winner_index=$(printf '%s\n' "$winner_indices" | jq -r --argjson offset "$winner_offset" '.[$offset]') +log "selection basis: quota-selected" +clean_profile_at "$winner_index" diff --git a/docs/architecture.md b/docs/architecture.md index ca79681..e24b8d0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -133,8 +133,8 @@ Ship tasks change projects and ship by project mode (`no-mistakes`, `direct-PR`, Crewmate and scout dispatch can stay on the static crewmate harness resolved by `config/crew-harness`, or it can use local dispatch profiles in `config/crew-dispatch.json`. The dispatch file is intentionally judgment-based: firstmate reads the natural-language rules at intake, chooses the best matching rule, resolves that rule directly or through a supported selector, and passes only concrete `--harness`, `--model`, and `--effort` axes to `fm-spawn.sh`. -The shell scripts validate the JSON shape and verified harness/effort combinations, and `fm-dispatch-select.sh` owns deterministic selector behavior, but they do not parse task intent or match the natural-language rules. -The session-start bootstrap step surfaces either the active rule block or a concise invalid-config line at startup. +The shell scripts validate the JSON shape and verified harness/effort combinations, and `fm-dispatch-select.sh` owns quota-aware array selection plus OS-backed random fallback, but they do not parse task intent or match the natural-language rules. +The session-start bootstrap step keeps valid dispatch configuration silent unless verbose facts are enabled and surfaces a concise invalid-config line when validation fails. When the file exists, `fm-spawn.sh` refuses crewmate and scout launches without an explicit harness, so `config/crew-harness` is only automatic when no dispatch profile file is active. Secondmate launches are exempt because they resolve the secondmate harness and any optional secondmate model or effort tokens instead. Unsupported effort values are still recorded in task meta when passed to `fm-spawn.sh`, but the launch template omits any effort flag that the selected harness does not accept. diff --git a/docs/configuration.md b/docs/configuration.md index 48c8ed4..f57ebef 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -208,24 +208,27 @@ This section is the single owner of the canonical schema and its per-field seman "why": "" } ], - "default": { "harness": "", "model": "", "effort": "" } + "default": [ + { "harness": "", "model": "", "effort": "" } + ] } ``` Per rule, `when` and `use` are required. -`use` may be a single profile object or an ordered array of profile objects; the single-object form stays fully backward-compatible, and every profile needs `harness`. -`use.model`, `use.effort`, and `why` are optional. -`select` is optional and currently supports `quota-balanced`. -Absent `select` means use the first array element, or the only object in the single-object form; the first array element is the deterministic tie-break and the ultimate fallback. -`default` is optional. +Both `use` and the optional top-level `default` accept either one profile object or a non-empty array of profile objects. +The single-object form stays fully backward-compatible, and every profile needs `harness`. +Profile `model` and `effort` fields and rule `why` are optional. An omitted model or effort means the selected harness uses its own default for that axis. +Every profile array is an implicit quota-aware choice and does not need a selector property. +`select: "quota-balanced"` remains accepted on rules for compatibility and has the same behavior as an implicit array choice. +If no dispatch rule fits, firstmate resolves `default` through the same object-or-array selection path before falling back to `config/crew-harness`. If a selected profile carries an effort value the chosen harness does not accept, `fm-spawn.sh` records the requested `effort=` in task meta for traceability but omits the launch flag, and bootstrap reports the invalid harness/effort pair as a `CREW_DISPATCH` diagnostic when it is visible in the file. -`quota-balanced` selection is deterministic and implemented by `bin/fm-dispatch-select.sh`, whose header owns the general-window rules, the 20 point stale-clear freshness margin, vendor-availability handling, and the degrade-to-first-element fallbacks; quota trouble never blocks dispatch. +Quota-aware selection is implemented by `bin/fm-dispatch-select.sh`, whose header owns provider and product mapping, relevant-window scoring, the stale-clear freshness margin, random tie-breaking, OS-backed random operational fallback, and safe selection-basis diagnostics. +Quota-data trouble never blocks dispatch, but malformed profile configuration remains an actionable validation error. See [`docs/examples/crew-dispatch.json`](examples/crew-dispatch.json) for a starting point to copy into local `config/crew-dispatch.json`. When the file exists, bootstrap validates it with `jq`. -Valid files stay silent by default; with `FM_BOOTSTRAP_VERBOSE_FACTS=1`, bootstrap emits `BOOTSTRAP_INFO: crew dispatch active config/crew-dispatch.json` plus one `BOOTSTRAP_INFO:` fact per rule and default profile. -Malformed JSON, an unverified harness, a malformed array profile, an unknown `select`, or an effort value unsupported by that harness is reported as `CREW_DISPATCH: invalid config/crew-dispatch.json - ...`; missing `jq` is reported through the normal `MISSING: jq` install-consent flow. -If no dispatch rule fits, firstmate uses the dispatch profile `default` when present, then falls back to `config/crew-harness`. +Valid files stay silent by default; with `FM_BOOTSTRAP_VERBOSE_FACTS=1`, bootstrap emits `BOOTSTRAP_INFO: crew dispatch active config/crew-dispatch.json`, one `BOOTSTRAP_INFO:` fact per rule, and one fact for the optional default profile set. +Malformed JSON, an empty or malformed rule/default array, an unverified harness, an unknown `select`, or an effort value unsupported by that harness is reported as `CREW_DISPATCH: invalid config/crew-dispatch.json - ...`; missing `jq` is reported through the normal `MISSING: jq` install-consent flow. Because the spawn backstop is gated by file presence, any fallback path after a missing match, validation error, or missing `jq` still passes a resolved harness explicitly until the file is fixed or removed. Secondmate homes inherit this file from the primary, so a secondmate's own crewmates apply the same dispatch profile behavior. @@ -236,7 +239,7 @@ It installs automatically supported tools only after you say go; manual-only too Required tools come in two parts: a universal toolchain every home needs regardless of backend, and a per-backend delta that follows the runtime backend actually resolved for this home. The universal toolchain is node, git, gh with GitHub auth via `gh auth login`, no-mistakes v1.31.2 or newer, gh-axi, chrome-devtools-axi, lavish-axi, compatible tasks-axi per "Backlog backend" above, and quota-axi. This section is the single owner of that universal toolchain list; backend guides' prerequisites point here and add only their backend-specific tools. -In that list, no-mistakes runs the validation pipeline, gh-axi, chrome-devtools-axi, and lavish-axi cover GitHub, browser, and rich-review operations, and tasks-axi plus quota-axi back backlog mutations and quota-balanced dispatch. +In that list, no-mistakes runs the validation pipeline, gh-axi, chrome-devtools-axi, and lavish-axi cover GitHub, browser, and rich-review operations, and tasks-axi plus quota-axi back backlog mutations and quota-aware array dispatch. The per-backend delta is required only for the backend resolved from `FM_BACKEND`, then `config/backend`, then runtime auto-detection, then default `tmux`, so a home is never told to install a tool an inactive backend or feature would need. That delta is owned in code by `fm_backend_required_tools` in `bin/fm-backend.sh`: the resolved backend's own session-provider CLI (`tmux`, `herdr`, `zellij`, `orca`, or `cmux`), `jq` for the JSON-emitting experimental adapters (`herdr`, `zellij`, `cmux`) whose spawn and liveness paths parse the backend's JSON output, and the `treehouse` worktree provider for every session-provider-only backend (`tmux`, `herdr`, `zellij`, `cmux`). Backend tool availability uses the adapter's own executable resolver, so bootstrap and spawn agree on supported non-`PATH` locations such as cmux's bundled CLI. @@ -247,7 +250,7 @@ When `config/crew-dispatch.json` exists, bootstrap also requires `jq` for dispat When X mode is opted in, bootstrap also requires `curl` and `jq` before arming the relay poll shim. `tasks-axi` and `quota-axi` are required bootstrap tools in every profile, the same class as `lavish-axi`. An absent or incompatible `tasks-axi` reports `MISSING: tasks-axi (install: npm install -g tasks-axi)`; when `config/backlog-backend` is not `manual` and compatible `tasks-axi` is on `PATH`, bootstrap stays silent and firstmate uses its verbs for routine backlog mutations, otherwise it hand-edits `data/backlog.md` until installation is approved and completed. -An absent `quota-axi` reports `MISSING: quota-axi (install: npm install -g quota-axi)`; `bin/fm-dispatch-select.sh` still degrades to the first profile at runtime when quota data is unavailable. +An absent `quota-axi` reports `MISSING: quota-axi (install: npm install -g quota-axi)`; `bin/fm-dispatch-select.sh` still selects uniformly from the valid candidate array with an OS-backed random source when quota data is unavailable. Bootstrap also reports a `TANGLE:` line when `FM_ROOT` is on a named non-default branch; follow the printed checkout remediation rather than treating it as an installable tool problem. In a read-only session that did not get the fleet lock, the same line is advisory and omits the checkout command. The locked session-start bootstrap step also runs a best-effort project clone refresh through `fm-fleet-sync.sh`. diff --git a/docs/examples/crew-dispatch.json b/docs/examples/crew-dispatch.json index 8aec196..886557f 100644 --- a/docs/examples/crew-dispatch.json +++ b/docs/examples/crew-dispatch.json @@ -16,9 +16,11 @@ { "harness": "claude", "model": "claude-sonnet-5", "effort": "high" }, { "harness": "codex", "model": "gpt-5.5", "effort": "high" } ], - "select": "quota-balanced", - "why": "Use a strong coding profile for broad design and implementation work." + "why": "Arrays are quota-aware automatically, so use a strong coding profile with the most available relevant quota." } ], - "default": { "harness": "codex", "model": "gpt-5.5", "effort": "medium" } + "default": [ + { "harness": "codex", "model": "gpt-5.5", "effort": "medium" }, + { "harness": "pi", "model": "anthropic/claude-sonnet-5", "effort": "medium" } + ] } diff --git a/docs/scripts.md b/docs/scripts.md index 1b11d23..16c52ff 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -37,7 +37,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-supervision-instructions.sh` | Render the session-start primary-harness supervision block or the one-line repair instruction | | `fm-home-seed.sh` | Transactionally provision a secondmate home and maintain `data/secondmates.md` | | `fm-spawn.sh` | Spawn crewmates, scouts, `id=repo` batches, and secondmates on the resolved harness and runtime backend | -| `fm-dispatch-select.sh` | Resolve a matched crew-dispatch rule to one concrete profile, owning `quota-balanced` selection | +| `fm-dispatch-select.sh` | Resolve a dispatch rule/default to one profile, owning quota-aware arrays and random fallback | | `fm-backend.sh` | Runtime-backend selection, meta helpers, selector resolution, and operation dispatch | | `fm-backend-hometag-lib.sh` | Shared per-installation home-tag derivation for zellij tab and cmux workspace titles | | `fm-composer-lib.sh` | Single fleet-wide owner of composer-content classification for all backends | diff --git a/tests/fm-bootstrap.test.sh b/tests/fm-bootstrap.test.sh index 851ba66..9e49d18 100755 --- a/tests/fm-bootstrap.test.sh +++ b/tests/fm-bootstrap.test.sh @@ -730,7 +730,7 @@ test_crew_dispatch_active_rules_are_verbose_bootstrap_info() { case_dir="$TMP_ROOT/dispatch-active" mkdir -p "$case_dir/home/config" printf '%s\n' manual > "$case_dir/home/config/backlog-backend" - printf '%s\n' '{"rules":[{"when":"fresh news","use":{"harness":"grok"},"why":"current context"},{"when":"big feature","use":[{"harness":"claude","model":"claude-sonnet-5","effort":"high"},{"harness":"codex","model":"gpt-5.5","effort":"high"}],"select":"quota-balanced"}],"default":{"harness":"claude","model":"haiku","effort":"low"}}' > "$case_dir/home/config/crew-dispatch.json" + printf '%s\n' '{"rules":[{"when":"fresh news","use":{"harness":"grok"},"why":"current context"},{"when":"big feature","use":[{"harness":"claude","model":"claude-sonnet-5","effort":"high"},{"harness":"codex","model":"gpt-5.5","effort":"high"}]},{"when":"legacy feature","use":[{"harness":"claude"},{"harness":"codex"}],"select":"quota-balanced"}],"default":[{"harness":"pi","model":"anthropic/claude-sonnet-5","effort":"high"},{"harness":"grok","model":"grok-4.5","effort":"high"}]}' > "$case_dir/home/config/crew-dispatch.json" fakebin=$(make_fake_toolchain "$case_dir") add_real_jq "$fakebin" @@ -741,7 +741,7 @@ test_crew_dispatch_active_rules_are_verbose_bootstrap_info() { out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$case_dir/home" FM_ROOT_OVERRIDE="$case_dir/home" \ FM_BOOTSTRAP_VERBOSE_FACTS=1 FM_FAKE_TREEHOUSE_LEASE_HELP=1 "$ROOT/bin/fm-bootstrap.sh") - expect=$'BOOTSTRAP_INFO: crew dispatch active config/crew-dispatch.json\nBOOTSTRAP_INFO: crew dispatch rule: fresh news -> grok\nBOOTSTRAP_INFO: crew dispatch rule: big feature -> quota-balanced[claude/claude-sonnet-5/high, codex/gpt-5.5/high]\nBOOTSTRAP_INFO: crew dispatch default: claude/haiku/low' + expect=$'BOOTSTRAP_INFO: crew dispatch active config/crew-dispatch.json\nBOOTSTRAP_INFO: crew dispatch rule: fresh news -> grok\nBOOTSTRAP_INFO: crew dispatch rule: big feature -> quota-balanced[claude/claude-sonnet-5/high, codex/gpt-5.5/high]\nBOOTSTRAP_INFO: crew dispatch rule: legacy feature -> quota-balanced[claude, codex]\nBOOTSTRAP_INFO: crew dispatch default: quota-balanced[pi/anthropic/claude-sonnet-5/high, grok/grok-4.5/high]' [ "$out" = "$expect" ] || fail "active dispatch verbose info block mismatch"$'\n'"expected: $expect"$'\n'"actual: $out" pass "bootstrap surfaces active crew-dispatch rules only as verbose BOOTSTRAP_INFO" } @@ -778,10 +778,18 @@ pi max effort is accepted^{"rules":[{"when":"deep coding","use":{"harness":"pi", unsupported opencode effort is flagged^{"rules":[{"when":"opencode work","use":{"harness":"opencode","model":"anthropic/claude-sonnet-4-5","effort":"high"}}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - invalid effort: opencode:high array use with quota-balanced is accepted^{"rules":[{"when":"big feature","use":[{"harness":"claude","model":"claude-sonnet-5","effort":"high"},{"harness":"codex","model":"gpt-5.5","effort":"high"}],"select":"quota-balanced"}]}^empty^ array use without select is accepted^{"rules":[{"when":"big feature","use":[{"harness":"claude"},{"harness":"codex"}]}]}^empty^ +one-element array use is accepted^{"rules":[{"when":"focused feature","use":[{"harness":"claude"}]}]}^empty^ +default array is accepted^{"default":[{"harness":"pi","model":"anthropic/claude-sonnet-5"},{"harness":"grok"}]}^empty^ +one-element default array is accepted^{"default":[{"harness":"codex"}]}^empty^ empty array use is flagged^{"rules":[{"when":"big feature","use":[]}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - each rule needs at least one use profile array profile without harness is flagged^{"rules":[{"when":"big feature","use":[{"model":"gpt-5.5"}]}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - each use profile needs harness +array profile with malformed model is flagged^{"rules":[{"when":"big feature","use":[{"harness":"codex","model":5}]}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - use profile model and effort must be non-empty strings when present unknown select is flagged^{"rules":[{"when":"big feature","use":[{"harness":"claude"},{"harness":"codex"}],"select":"mystery"}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - unknown select: mystery array profile unsupported effort is flagged^{"rules":[{"when":"big feature","use":[{"harness":"codex","effort":"max"}]}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - invalid effort: codex:max +empty default array is flagged^{"default":[]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - default needs at least one profile +non-object default array entry is flagged^{"default":["codex"]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - each default profile must be an object +default array profile without harness is flagged^{"default":[{"model":"gpt-5.5"}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - each default profile needs harness +default array malformed effort is flagged^{"default":[{"harness":"codex","effort":3}]}^exact^CREW_DISPATCH: invalid config/crew-dispatch.json - default profile model and effort must be non-empty strings when present ROWS pass "bootstrap validates crew-dispatch.json and reports malformed or unverified configs" } diff --git a/tests/fm-dispatch-select.test.sh b/tests/fm-dispatch-select.test.sh index a9ae5b7..fdcae4b 100755 --- a/tests/fm-dispatch-select.test.sh +++ b/tests/fm-dispatch-select.test.sh @@ -1,5 +1,24 @@ #!/usr/bin/env bash -# Behavior tests for deterministic crew-dispatch profile selection. +# Behavior tests for quota-aware crew-dispatch profile selection. +# +# End-user reproduction before the fix: +# - Initiating input: a non-empty profile array in rule use or top-level default. +# - Expected: startup accepts both locations and dispatch consults quota-axi. +# - Observed: startup rejected a default array, while a rule array without +# select silently chose its first element without calling quota-axi. +# - Masking condition: a single profile object worked, and an explicit +# quota-balanced rule array took the quota path. +# - Visible symptom: an actionable-looking startup invalid-config line for a +# valid default array, or first-profile dispatch despite better usable quota. +# - Earliest divergence: bootstrap restricted default to object while use had an +# array normalizer; selector gated quota lookup on explicit select rather than +# the already-normalized input being an array. +# - History: 7a42707 added rule arrays and explicit quota-balanced selection; +# 8cd90fe moved the contract owner without changing that asymmetric behavior. +# - Smallest counterfactual: adding select changed a rule array to quota-aware +# selection but could not make the same array valid under default. +# - Disconfirming evidence: object defaults, object rule uses, and explicit +# quota-balanced rule arrays all followed their proven paths successfully. set -u # shellcheck source=tests/lib.sh @@ -8,12 +27,17 @@ set -u BASE_PATH=${FM_TEST_BASE_PATH:-/usr/bin:/bin:/usr/sbin:/sbin} TMP_ROOT=$(fm_test_tmproot fm-dispatch-select-tests) mkdir -p "$TMP_ROOT" +RANDOM_ZERO="$TMP_ROOT/random-zero" +RANDOM_ONE="$TMP_ROOT/random-one" +printf '\000\000\000\000' > "$RANDOM_ZERO" +printf '\001\001\001\001' > "$RANDOM_ONE" write_quota() { local file=$1 claude_status=$2 claude_five=$3 claude_week=$4 codex_status=$5 codex_five=$6 codex_week=$7 mkdir -p "$(dirname "$file")" cat > "$file" <"$TMP_ROOT/higher.err") + err=$(cat "$TMP_ROOT/higher.err") + assert_profile "$out" '{"harness":"codex","model":"gpt-5.5","effort":"high"}' "higher-min provider should win" + assert_contains "$err" "selection basis: quota-selected" "quota selection basis was not exposed" + pass "every profile array implicitly picks the least constrained scorable provider" } -test_exact_tie_uses_first_profile() { +test_rule_array_without_select_invokes_quota_axi() { + local fakebin marker out rule + fakebin=$(fm_fakebin "$TMP_ROOT/implicit-command") + marker="$TMP_ROOT/implicit-command/called" + cat > "$fakebin/quota-axi" < '$marker' +cat <<'JSON' +{"schemaVersion":2,"providers":[{"provider":"claude","state":{"status":"fresh"},"windows":[{"id":"five_hour","kind":"session","percentRemaining":10}]},{"provider":"codex","state":{"status":"fresh"},"windows":[{"id":"five_hour","kind":"session","percentRemaining":90}]}]} +JSON +SH + chmod +x "$fakebin/quota-axi" + rule='{"when":"big work","use":[{"harness":"claude"},{"harness":"codex"}]}' + out=$(PATH="$fakebin:$BASE_PATH" FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ZERO" "$ROOT/bin/fm-dispatch-select.sh" "$rule" 2>/dev/null) + assert_profile "$out" '{"harness":"codex"}' "implicit rule array should use quota data" + assert_contains "$(cat "$marker")" "--json" "implicit array did not invoke quota-axi --json" + pass "rule arrays need no select property to invoke installed quota-axi" +} + +test_legacy_explicit_selector_stays_compatible() { local quota out + quota="$TMP_ROOT/legacy.json" + write_quota "$quota" fresh 90 80 fresh 70 60 + out=$(FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ZERO" "$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles" 2>/dev/null) + assert_profile "$out" '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' "legacy explicit selector changed behavior" + + out=$(FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ZERO" "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" \ + '{"when":"big work","use":[{"harness":"claude"},{"harness":"codex"}],"select":"quota-balanced"}' 2>/dev/null) + assert_profile "$out" '{"harness":"claude"}' "legacy rule selector changed behavior" + pass "legacy select quota-balanced forms remain compatible" +} + +test_equal_winners_use_os_random_tie_break() { + local quota first second quota="$TMP_ROOT/tie.json" write_quota "$quota" fresh 90 50 fresh 60 50 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "exact tie should pick first profile, got: $out" - pass "quota-balanced exact tie uses the first ordered profile" + first=$(FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ZERO" "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" "$profiles" 2>/dev/null) + second=$(FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ONE" "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" "$profiles" 2>/dev/null) + assert_profile "$first" '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' "zero random fixture should choose first tie winner" + assert_profile "$second" '{"harness":"codex","model":"gpt-5.5","effort":"high"}' "nonzero random fixture should choose second tie winner" + pass "equal quota winners use the OS-backed random tie-break" } -test_quota_missing_falls_back_to_first() { - local fakebin out err status - fakebin=$(fm_fakebin "$TMP_ROOT/missing") - out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced "$profiles" 2>"$TMP_ROOT/missing.err") - status=$? - err=$(cat "$TMP_ROOT/missing.err") - expect_code 0 "$status" "missing quota-axi should not fail dispatch" - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "missing quota-axi should fall back to first, got: $out" - assert_contains "$err" "quota-axi missing" "missing quota-axi fallback should be logged" - pass "quota-axi missing falls back to the first profile and logs" -} - -test_quota_error_falls_back_to_first() { - local fakebin out err status - fakebin=$(fm_fakebin "$TMP_ROOT/error") - cat > "$fakebin/quota-axi" <<'SH' -#!/usr/bin/env bash -exit 42 -SH - chmod +x "$fakebin/quota-axi" - out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced "$profiles" 2>"$TMP_ROOT/error.err") - status=$? - err=$(cat "$TMP_ROOT/error.err") - expect_code 0 "$status" "quota-axi error should not fail dispatch" - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "quota-axi error should fall back to first, got: $out" - assert_contains "$err" "quota-axi exited 42" "quota-axi error fallback should be logged" - pass "quota-axi non-zero exit falls back to the first profile and logs" +test_provider_and_product_mapping_through_wrappers() { + local quota out + quota="$TMP_ROOT/routes.json" + cat > "$quota" <<'JSON' +{ + "schemaVersion": 2, + "providers": [ + {"provider":"claude","state":{"status":"fresh"},"windows":[{"id":"five_hour","kind":"session","percentRemaining":45},{"id":"seven_day","kind":"weekly","percentRemaining":40}]}, + {"provider":"codex","state":{"status":"fresh"},"windows":[{"id":"five_hour","kind":"session","percentRemaining":55},{"id":"weekly","kind":"weekly","percentRemaining":50}]}, + {"provider":"grok","state":{"status":"fresh"},"windows":[ + {"id":"credits","kind":"credits","percentRemaining":1}, + {"id":"product:api","kind":"credits","percentRemaining":75}, + {"id":"product:grok_build","kind":"credits","percentRemaining":25} + ]} + ] } +JSON + out=$(FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ZERO" "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" \ + '[{"harness":"claude"},{"harness":"pi","model":"openai-codex/gpt-5.5"}]' 2>/dev/null) + assert_profile "$out" '{"harness":"pi","model":"openai-codex/gpt-5.5"}' "Pi OpenAI Codex route was not scored as Codex" -test_bad_quota_json_falls_back_to_first() { - local quota out err - quota="$TMP_ROOT/bad.json" - printf '%s\n' 'not-json' > "$quota" - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles" 2>"$TMP_ROOT/bad.err") - err=$(cat "$TMP_ROOT/bad.err") - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "bad quota JSON should fall back to first, got: $out" - assert_contains "$err" "unparseable JSON" "bad quota JSON fallback should be logged" - pass "unparseable quota JSON falls back to the first profile and logs" + out=$(FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ZERO" "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" \ + '[{"harness":"pi","model":"anthropic/claude-sonnet-5"},{"harness":"codex"}]' 2>/dev/null) + assert_profile "$out" '{"harness":"codex"}' "Pi Anthropic route was not scored as Claude" + + out=$(FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ZERO" "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" \ + '[{"harness":"pi","model":"xai/grok-4.5"},{"harness":"grok","model":"grok-4.5"}]' 2>/dev/null) + assert_profile "$out" '{"harness":"pi","model":"xai/grok-4.5"}' "Pi xAI API should use product:api rather than Grok Build or aggregate credits" + + out=$(FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ZERO" "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" \ + '[{"harness":"grok"},{"harness":"claude"}]' 2>/dev/null) + assert_profile "$out" '{"harness":"claude"}' "direct Grok should use product:grok_build" + pass "direct and Pi-wrapped candidates map to consumed Claude, Codex, xAI API, and Grok Build quota" } -test_stale_with_cache_needs_clear_margin_to_beat_fresh() { +test_most_constrained_relevant_window_scores_candidate() { + local quota out + quota="$TMP_ROOT/scoped.json" + cat > "$quota" <<'JSON' +{"schemaVersion":2,"providers":[ + {"provider":"claude","state":{"status":"fresh"},"windows":[ + {"id":"five_hour","kind":"session","percentRemaining":90}, + {"id":"seven_day","kind":"weekly","percentRemaining":80}, + {"id":"model:fable","label":"Fable week","kind":"model","percentRemaining":5} + ]}, + {"provider":"codex","state":{"status":"fresh"},"windows":[ + {"id":"five_hour","kind":"session","percentRemaining":30}, + {"id":"weekly","kind":"weekly","percentRemaining":30}, + {"id":"code_review_five_hour","label":"code review session","kind":"session","percentRemaining":1}, + {"id":"code_review_weekly","label":"code review week","kind":"weekly","percentRemaining":1}, + {"id":"model:other:5h","label":"Unrelated preview session","kind":"model","percentRemaining":1} + ]} +]} +JSON + out=$(FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ZERO" "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" \ + '[{"harness":"claude","model":"claude-fable-5"},{"harness":"codex","model":"gpt-5.5"}]' 2>/dev/null) + assert_profile "$out" '{"harness":"codex","model":"gpt-5.5"}' "matching model window was not included or unrelated model window was included" + pass "candidate score uses its most constrained general or matching model quota window" +} + +test_grok_aggregate_fallback_requires_no_product_windows() { + local quota out + quota="$TMP_ROOT/grok-partial-products.json" + cat > "$quota" <<'JSON' +{"schemaVersion":2,"providers":[ + {"provider":"grok","state":{"status":"fresh"},"windows":[ + {"id":"credits","kind":"credits","percentRemaining":100}, + {"id":"product:grok_build","kind":"credits","percentRemaining":90} + ]}, + {"provider":"claude","state":{"status":"fresh"},"windows":[ + {"id":"five_hour","kind":"session","percentRemaining":5}, + {"id":"seven_day","kind":"weekly","percentRemaining":5} + ]} +]} +JSON + out=$(FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ZERO" "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" \ + '[{"harness":"pi","model":"xai/grok-4.5"},{"harness":"claude"}]' 2>/dev/null) + assert_profile "$out" '{"harness":"claude"}' "xAI API route used aggregate credits despite exposed product windows" + pass "Grok aggregate credits are used only when product windows are absent" +} + +test_stale_cache_needs_clear_margin_to_beat_fresh() { local quota out quota="$TMP_ROOT/stale-margin.json" write_quota "$quota" stale 85 70 fresh 65 60 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") - [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ - || fail "fresh vendor should win when stale lead is below margin, got: $out" + out=$(FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ZERO" "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" "$profiles" 2>/dev/null) + assert_profile "$out" '{"harness":"codex","model":"gpt-5.5","effort":"high"}' "fresh provider should win below stale margin" write_quota "$quota" stale 90 85 fresh 65 60 - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "stale vendor should win when lead clears margin, got: $out" - pass "stale cached quota is usable only when it clears the documented margin over fresh" + out=$(FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ZERO" "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" "$profiles" 2>/dev/null) + assert_profile "$out" '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' "stale provider should win after clearing margin" + pass "stale cached quota retains the documented freshness margin" } -test_vendor_absent_or_unusable_falls_back_conservatively() { - local quota out err - quota="$TMP_ROOT/absent.json" +test_partial_quota_data_prefers_scorable_candidate() { + local quota out + quota="$TMP_ROOT/partial.json" cat > "$quota" <<'JSON' -{ - "providers": [ - { - "provider": "codex", - "state": { "status": "fresh" }, - "windows": [ - { "id": "five_hour", "kind": "session", "percentRemaining": 40 }, - { "id": "weekly", "kind": "weekly", "percentRemaining": 50 } - ] - } - ] -} +{"schemaVersion":2,"providers":[{"provider":"codex","state":{"status":"fresh"},"windows":[{"id":"five_hour","kind":"session","percentRemaining":4}]}]} JSON - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles") - [ "$out" = '{"harness":"codex","model":"gpt-5.5","effort":"high"}' ] \ - || fail "available candidate should win over absent vendor, got: $out" + out=$(FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ZERO" "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" "$profiles" 2>/dev/null) + assert_profile "$out" '{"harness":"codex","model":"gpt-5.5","effort":"high"}' "unscorable first candidate beat usable Codex data" + pass "partial quota data picks the best scorable candidate instead of an unscorable candidate" +} - cat > "$quota" <<'JSON' -{ "providers": [] } -JSON - out=$("$ROOT/bin/fm-dispatch-select.sh" --select quota-balanced --quota-json "$quota" "$profiles" 2>"$TMP_ROOT/none.err") - err=$(cat "$TMP_ROOT/none.err") - [ "$out" = '{"harness":"claude","model":"claude-sonnet-5","effort":"high"}' ] \ - || fail "no usable vendors should fall back to first, got: $out" - assert_contains "$err" "no usable quota windows" "no usable vendor fallback should be logged" - pass "absent or unusable vendors resolve to an available candidate or the first fallback" -} - -test_backward_compatible_first_selection() { - local fakebin marker out single array_rule - fakebin=$(fm_fakebin "$TMP_ROOT/no-call") - marker="$TMP_ROOT/quota-called" +assert_random_fallback_chooses_second() { + local out_file=$1 err_file=$2 + shift 2 + FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ONE" "$@" >"$out_file" 2>"$err_file" + assert_profile "$(cat "$out_file")" '{"harness":"codex","model":"gpt-5.5","effort":"high"}' "random fallback fixture should choose the second candidate" + assert_contains "$(cat "$err_file")" "selection basis: random fallback" "random fallback basis was not exposed" +} + +test_operational_quota_failures_use_uniform_random_fallback() { + local fakebin quota + fakebin=$(fm_fakebin "$TMP_ROOT/missing") + assert_random_fallback_chooses_second "$TMP_ROOT/missing.out" "$TMP_ROOT/missing.err" \ + env PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" "$profiles" + assert_contains "$(cat "$TMP_ROOT/missing.err")" "quota-axi missing" "missing quota-axi reason was not logged" + + fakebin=$(fm_fakebin "$TMP_ROOT/error") + cat > "$fakebin/quota-axi" <<'SH' +#!/usr/bin/env bash +exit 42 +SH + chmod +x "$fakebin/quota-axi" + assert_random_fallback_chooses_second "$TMP_ROOT/error.out" "$TMP_ROOT/error.err" \ + env PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" "$profiles" + assert_contains "$(cat "$TMP_ROOT/error.err")" "quota-axi exited 42" "quota-axi error reason was not logged" + + quota="$TMP_ROOT/bad.json" + printf '%s\n' not-json > "$quota" + assert_random_fallback_chooses_second "$TMP_ROOT/bad.out" "$TMP_ROOT/bad.err" \ + "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" "$profiles" + assert_contains "$(cat "$TMP_ROOT/bad.err")" "unparseable JSON" "bad quota JSON reason was not logged" + + printf '%s\n' '{"schemaVersion":2,"providers":[]}' > "$quota" + assert_random_fallback_chooses_second "$TMP_ROOT/empty.out" "$TMP_ROOT/empty.err" \ + "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" "$profiles" + assert_contains "$(cat "$TMP_ROOT/empty.err")" "no usable quota windows" "wholly unusable quota reason was not logged" + pass "missing, failed, malformed, and wholly unusable quota data use OS-backed random fallback" +} + +test_single_profile_and_one_element_array() { + local fakebin marker out err + fakebin=$(fm_fakebin "$TMP_ROOT/single") + marker="$TMP_ROOT/single/called" cat > "$fakebin/quota-axi" < '$marker' @@ -161,26 +277,51 @@ exit 1 SH chmod +x "$fakebin/quota-axi" - single='{"harness":"grok","model":"grok-4","effort":"high"}' - out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" "$single") - [ "$out" = '{"harness":"grok","model":"grok-4","effort":"high"}' ] \ - || fail "single-object use should resolve to itself, got: $out" - - array_rule='{"when":"big work","use":[{"harness":"claude","effort":"high"},{"harness":"codex","effort":"high"}]}' - out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" "$array_rule") - [ "$out" = '{"harness":"claude","effort":"high"}' ] \ - || fail "array without select should resolve to first, got: $out" - [ ! -e "$marker" ] || fail "quota-axi should not be called without quota-balanced select" - pass "single-object use and no-select arrays preserve first-profile selection" -} - -test_higher_min_vendor_wins -test_exact_tie_uses_first_profile -test_quota_missing_falls_back_to_first -test_quota_error_falls_back_to_first -test_bad_quota_json_falls_back_to_first -test_stale_with_cache_needs_clear_margin_to_beat_fresh -test_vendor_absent_or_unusable_falls_back_conservatively -test_backward_compatible_first_selection + out=$(PATH="$fakebin:$BASE_PATH" "$ROOT/bin/fm-dispatch-select.sh" '{"harness":"grok","model":"grok-4.5","effort":"high"}' 2>/dev/null) + assert_profile "$out" '{"harness":"grok","model":"grok-4.5","effort":"high"}' "single profile object should resolve to itself" + [ ! -e "$marker" ] || fail "single profile object should not invoke quota-axi" + + out=$(PATH="$fakebin:$BASE_PATH" FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ONE" "$ROOT/bin/fm-dispatch-select.sh" \ + '[{"harness":"grok","model":"grok-4.5","effort":"high"}]' 2>"$TMP_ROOT/one.err") + err=$(cat "$TMP_ROOT/one.err") + assert_profile "$out" '{"harness":"grok","model":"grok-4.5","effort":"high"}' "one-element array should remain selectable" + [ -e "$marker" ] || fail "one-element array should invoke quota-axi" + assert_contains "$err" "selection basis: random fallback" "one-element operational fallback basis was not logged" + pass "single objects remain backward compatible and one-element arrays remain quota-aware" +} + +test_malformed_profile_arrays_are_validation_errors() { + local body expect out status n + n=0 + while IFS='^' read -r body expect; do + n=$((n + 1)) + out=$(FM_DISPATCH_RANDOM_SOURCE="$RANDOM_ONE" "$ROOT/bin/fm-dispatch-select.sh" "$body" 2>&1) + status=$? + expect_code 2 "$status" "malformed profile array should exit 2" + assert_contains "$out" "$expect" "malformed profile array did not explain validation error" + assert_not_contains "$out" "random fallback" "malformed profile array incorrectly used operational fallback" + done <<'ROWS' +[]^must not be empty +["claude"]^must be an object +[{"model":"claude-sonnet-5"}]^needs a non-empty harness +[{"harness":"claude","model":3}]^model must be a non-empty string +[{"harness":"spaceship"}]^contains an unverified harness +[{"harness":"codex","effort":"max"}]^contains an unsupported harness/effort pair +ROWS + pass "malformed arrays stay actionable validation errors and never enter random fallback" +} + +test_implicit_array_picks_higher_min_provider +test_rule_array_without_select_invokes_quota_axi +test_legacy_explicit_selector_stays_compatible +test_equal_winners_use_os_random_tie_break +test_provider_and_product_mapping_through_wrappers +test_most_constrained_relevant_window_scores_candidate +test_grok_aggregate_fallback_requires_no_product_windows +test_stale_cache_needs_clear_margin_to_beat_fresh +test_partial_quota_data_prefers_scorable_candidate +test_operational_quota_failures_use_uniform_random_fallback +test_single_profile_and_one_element_array +test_malformed_profile_arrays_are_validation_errors echo "# all fm-dispatch-select tests passed" diff --git a/tests/fm-spawn-dispatch-profile.test.sh b/tests/fm-spawn-dispatch-profile.test.sh index d549c9b..6ae9b67 100755 --- a/tests/fm-spawn-dispatch-profile.test.sh +++ b/tests/fm-spawn-dispatch-profile.test.sh @@ -347,6 +347,40 @@ test_pi_threads_model_and_max_effort() { pass "pi receives --model and --thinking max profile flags" } +test_quota_selected_default_array_reaches_spawn() { + local rec id quota random selected diagnostic harness model effort out status launch + id=profile-selected-default-z17 + rec=$(make_spawn_case profile-selected-default claude "$id") + read_case_record "$rec" + cat > "$HOME_DIR/config/crew-dispatch.json" <<'JSON' +{"default":[{"harness":"claude","model":"claude-sonnet-5","effort":"low"},{"harness":"codex","model":"gpt-5.5","effort":"high"}]} +JSON + quota="$CASE_DIR/quota.json" + random="$CASE_DIR/random" + printf '\000\000\000\000' > "$random" + cat > "$quota" <<'JSON' +{"schemaVersion":2,"providers":[{"provider":"claude","state":{"status":"fresh"},"windows":[{"id":"five_hour","kind":"session","percentRemaining":10}]},{"provider":"codex","state":{"status":"fresh"},"windows":[{"id":"five_hour","kind":"session","percentRemaining":90}]}]} +JSON + + selected=$(FM_DISPATCH_RANDOM_SOURCE="$random" "$ROOT/bin/fm-dispatch-select.sh" --quota-json "$quota" \ + "$(jq -c .default "$HOME_DIR/config/crew-dispatch.json")" 2>"$CASE_DIR/selection.err") + diagnostic=$(cat "$CASE_DIR/selection.err") + harness=$(printf '%s\n' "$selected" | jq -r .harness) + model=$(printf '%s\n' "$selected" | jq -r .model) + effort=$(printf '%s\n' "$selected" | jq -r .effort) + out=$(run_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" \ + "$id" "$PROJ_DIR" --harness "$harness" --model "$model" --effort "$effort") + status=$? + + expect_code 0 "$status" "quota-selected default-array profile should reach spawn" + assert_contains "$diagnostic" "selection basis: quota-selected" "selection did not expose its quota basis" + assert_meta_profile "$HOME_DIR/state/$id.meta" codex gpt-5.5 high + launch=$(cat "$LAUNCH_LOG") + assert_contains "$launch" "codex --model 'gpt-5.5' -c 'model_reasoning_effort=\"high\"'" \ + "quota-selected default profile did not reach the concrete launch" + pass "top-level default array resolves through quota selection into the real spawn path" +} + test_batch_forwards_shared_profile_flags() { local rec id1 id2 out status id1=profile-batch-a-z9 @@ -398,6 +432,7 @@ test_grok_omits_invalid_max_reasoning_effort test_grok_omits_invalid_xhigh_reasoning_effort test_opencode_threads_model_and_ignores_effort_axis test_pi_threads_model_and_max_effort +test_quota_selected_default_array_reaches_spawn test_batch_forwards_shared_profile_flags test_active_dispatch_profile_does_not_block_secondmate_launch From 593e3a2c385ddce183fdb0c1fba3231984ac5fed Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:32:38 -0700 Subject: [PATCH 04/34] Add built-in ahoy recap skill (#873) --- .agents/skills/ahoy/SKILL.md | 31 +++++++++++++ README.md | 1 + tests/fm-captain-translation-contract.test.sh | 46 +++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 .agents/skills/ahoy/SKILL.md diff --git a/.agents/skills/ahoy/SKILL.md b/.agents/skills/ahoy/SKILL.md new file mode 100644 index 0000000..40d89be --- /dev/null +++ b/.agents/skills/ahoy/SKILL.md @@ -0,0 +1,31 @@ +--- +name: ahoy +description: Recap only the visible session events since the prior real captain message when the captain explicitly invokes /ahoy, with a Bearings fallback when /ahoy is the session's first real captain message. +user-invocable: true +metadata: + internal: true +--- + +# ahoy + +Give the captain a concise session-only recap without gathering fresh state. + +1. Inspect only conversation or session history already visible to the current first mate. +2. Find the most recent real captain-authored message before the current `/ahoy` invocation. + Use Firstmate's existing distinction between captain input and internal or synthetic notifications. + System, developer, tool, watcher, guard, away-mode, and other injected operational messages are not captain messages. +3. If no prior real captain message exists, load [`../bearings/SKILL.md`](../bearings/SKILL.md) and follow it exactly. + Bearings alone owns its gathering, artifact, and response contract. + Do not restate that contract or combine a session recap with Bearings output. +4. If a prior real captain message exists, recap only what happened after that message and before the current invocation. + Include concrete outcomes, landed work, failures, decisions made, new decisions needed, and work still running only when those events appear in visible session history. + Use captain-facing outcome language and preserve every full PR URL present in that interval. +5. The normal recap branch is session-history-only. + Do not call Bearings, shell commands, fleet snapshots, status readers, GitHub or browser APIs, tools, or file reads or writes. + Create no report, persist nothing, and do not guess current live state beyond the last visible event. +6. If nothing happened after the previous captain message, say so directly in one sentence. + +The current `/ahoy` message is outside the recap interval. +A previous `/ahoy` is a real captain message and may be the next interval boundary. +If context compaction makes the prior boundary unavailable, state that the exact session boundary is unavailable and summarize only visibly supported events. +Do not silently invoke Bearings unless this is genuinely the first real captain message. diff --git a/README.md b/README.md index 6c958d8..52d69cf 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,7 @@ Claude and grok use the slash form shown here; codex uses the same names with `$ | Skill | What it does | | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------- | | `/afk` | Enter away-mode supervision: the sub-supervisor self-handles routine notifications in bash, escalates captain-relevant events and bounded declared-external-wait rechecks as batched digests, and actively alerts if delivery gets stuck while you step away | +| `/ahoy` | Recap only visible session events since the prior real captain message, falling back to Bearings when invoked as the session's first real captain message | | `/bearings` | Generate a standalone current-status report from bounded local fleet and registered-secondmate state, with live PR enrichment only when requested, written to a dated file in `data/` and surfaced concisely in chat; read-mostly, mutates no task state | | `/updatefirstmate` | Self-update the running firstmate and its secondmates to the latest from origin with fast-forward-only pulls, then re-read instructions and nudge secondmates | | `/stow` | Sweep the session for uncaptured durable knowledge, route each finding to its disk home per AGENTS.md, file undone next steps to the backlog, and report what is now safe to reset | diff --git a/tests/fm-captain-translation-contract.test.sh b/tests/fm-captain-translation-contract.test.sh index 6ac57d8..fb3995d 100755 --- a/tests/fm-captain-translation-contract.test.sh +++ b/tests/fm-captain-translation-contract.test.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Static regression tests for the captain-facing plain-English translation # contract owned by AGENTS.md section 9. +# shellcheck disable=SC2016 set -u # shellcheck source=tests/lib.sh @@ -15,6 +16,8 @@ HARNESS="$ROOT/.agents/skills/harness-adapters/SKILL.md" CODEXAPP="$ROOT/.agents/skills/firstmate-codexapp/SKILL.md" FMX="$ROOT/.agents/skills/fmx-respond/SKILL.md" UPDATE="$ROOT/.agents/skills/updatefirstmate/SKILL.md" +AHOY="$ROOT/.agents/skills/ahoy/SKILL.md" +README="$ROOT/README.md" section_9() { awk ' @@ -138,6 +141,46 @@ test_section_9_owner_is_not_duplicated_into_skills() { pass "skills cross-reference section 9 instead of duplicating the mapping list" } +test_ahoy_is_an_internal_user_invocable_skill() { + assert_present "$AHOY" "ahoy skill is missing" + assert_grep 'name: ahoy' "$AHOY" "ahoy skill metadata has the wrong name" + assert_grep 'user-invocable: true' "$AHOY" "ahoy skill is not user-invocable" + assert_grep ' internal: true' "$AHOY" "ahoy skill is not internal" + [ ! -e "$ROOT/skills/ahoy" ] || fail "ahoy must not exist in the public installer-facing skills directory" + pass "ahoy is internal, user-invocable, and absent from public skills" +} + +test_ahoy_readme_uses_cross_harness_convention() { + assert_grep 'Claude and grok use the slash form shown here; codex uses the same names with `$`' "$README" \ + "README lost the cross-harness slash and dollar convention" + assert_grep '| `/ahoy`' "$README" "README built-in skills table does not list /ahoy" + pass "README lists ahoy under the shared cross-harness invocation convention" +} + +test_ahoy_owns_only_the_visible_session_recap() { + assert_grep '[`../bearings/SKILL.md`](../bearings/SKILL.md)' "$AHOY" \ + "first-message fallback does not delegate to Bearings by relative pointer" + assert_grep 'If no prior real captain message exists' "$AHOY" \ + "ahoy does not limit Bearings fallback to the first real captain message" + assert_grep 'System, developer, tool, watcher, guard, away-mode, and other injected operational messages are not captain messages.' "$AHOY" \ + "ahoy incorrectly treats synthetic operational messages as captain messages" + assert_grep 'The normal recap branch is session-history-only.' "$AHOY" \ + "later ahoy invocation is not explicitly session-history-only" + assert_grep 'Do not call Bearings, shell commands, fleet snapshots, status readers, GitHub or browser APIs, tools, or file reads or writes.' "$AHOY" \ + "normal recap does not prohibit fresh fleet, file, and tool reads" + assert_grep 'do not guess current live state beyond the last visible event' "$AHOY" \ + "normal recap may falsely claim a live snapshot" + assert_grep 'If context compaction makes the prior boundary unavailable' "$AHOY" \ + "ahoy does not disclose an unavailable compacted boundary" + assert_grep 'summarize only visibly supported events' "$AHOY" \ + "compacted fallback may invent unsupported events" + assert_no_grep 'fm-bearings-snapshot.sh' "$AHOY" \ + "ahoy copied Bearings gathering mechanics instead of referencing its owner" + assert_no_grep "Captain's Call" "$AHOY" \ + "ahoy copied Bearings response contract instead of referencing its owner" + pass "ahoy delegates first-message fallback and keeps later recaps visible-session-only" +} + test_section_9_owns_positive_translation_contract test_scout_remains_allowed_house_vocabulary test_compressed_safety_labels_have_plain_renderings @@ -145,3 +188,6 @@ test_mapping_list_covers_high_risk_internal_families test_verbatim_internal_evidence_is_rejected_from_chat test_outward_facing_skill_points_reference_section_9_owner test_section_9_owner_is_not_duplicated_into_skills +test_ahoy_is_an_internal_user_invocable_skill +test_ahoy_readme_uses_cross_harness_convention +test_ahoy_owns_only_the_visible_session_recap From 554983438606101316cb23726bdf80d4ae2f9a6b Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:35:20 -0700 Subject: [PATCH 05/34] fix: preserve trustworthy Bearings data in partial snapshots (#875) * fix: preserve mixed Bearings projections * no-mistakes(review): Enforce strict invalidity precedence for partial snapshots * no-mistakes(review): Enforce ownership for unknown child metadata * no-mistakes(document): Document partial structured Bearings projections * no-mistakes: apply CI fixes * no-mistakes: apply CI fixes --- .agents/skills/bearings/SKILL.md | 3 +- .github/workflows/ci.yml | 8 +- bin/fm-bearings-snapshot.sh | 42 ++-- bin/fm-fleet-snapshot.sh | 170 ++++++++++--- bin/fm-install-shellcheck.sh | 12 +- docs/architecture.md | 3 +- docs/decision-hold-lifecycle.md | 9 +- tests/fm-bearings-snapshot.test.sh | 364 +++++++++++++++++++++++++++ tests/fm-fleet-snapshot-view.test.sh | 94 +++++++ tests/fm-lint.test.sh | 57 +++++ 10 files changed, 696 insertions(+), 66 deletions(-) diff --git a/.agents/skills/bearings/SKILL.md b/.agents/skills/bearings/SKILL.md index 889009c..2365b92 100644 --- a/.agents/skills/bearings/SKILL.md +++ b/.agents/skills/bearings/SKILL.md @@ -66,7 +66,8 @@ Rules that keep the contract unambiguous: - Recently Landed always renders the bounded current baseline, even when the same completions appeared in an earlier report. - The four buckets are mutually exclusive, so every item is forced into exactly one: needs-your-action is Captain's Call, done is Recently Landed, self-progressing is Underway, and not-yet-started work or an action-free fleet-integrity warning is Charted Next. - The strict boundary keeps action-free items OUT of Captain's Call: a working or validating task, a queued item blocked on another task or a date, landed work, a completed scout's report pointer, a declared `paused:` external wait, and a bare recorded PR with no merge-ready signal each belong to one of the other three sections, never Captain's Call. -- A secondmate appears Underway only for `active_child_work`; `externally_held` belongs in Charted Next, and `unknown` belongs there as an unavailable-state gate unless its reason requires the captain's action. +- A secondmate's own row appears Underway only for `active_child_work`; `externally_held` belongs in Charted Next, and `unknown` belongs there as an unavailable-state gate unless its reason requires the captain's action. +- Do not suppress separately projected decisions, landed records, or gates from a `partial-structured` home merely because that secondmate's own row is `unknown`. - The chat follows `AGENTS.md` section 9 and carries one scannable line per item, each PR as the full `https://...` URL; detailed decisions, plans, full gate reasons, and evidence live only in the report file, which the chat links to, so the chat stays materially shorter than that file. ## Tone and content rules diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62dabea..c04c096 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -327,16 +327,16 @@ jobs: snapshot_output=$(/bin/bash tests/fm-fleet-snapshot-view.test.sh) printf '%s\n' "$snapshot_output" snapshot_count=$(printf '%s\n' "$snapshot_output" | grep -c '^ok - ') - [ "$snapshot_count" -eq 14 ] || { - echo "::error::expected 14 snapshot/fleet-view tests, got $snapshot_count" + [ "$snapshot_count" -eq 15 ] || { + echo "::error::expected 15 snapshot/fleet-view tests, got $snapshot_count" exit 1 } bearings_output=$(/bin/bash tests/fm-bearings-snapshot.test.sh) printf '%s\n' "$bearings_output" bearings_count=$(printf '%s\n' "$bearings_output" | grep -c '^ok - ') - [ "$bearings_count" -eq 39 ] || { - echo "::error::expected 39 Bearings tests, got $bearings_count" + [ "$bearings_count" -eq 42 ] || { + echo "::error::expected 42 Bearings tests, got $bearings_count" exit 1 } diff --git a/bin/fm-bearings-snapshot.sh b/bin/fm-bearings-snapshot.sh index a71523c..7564f9f 100755 --- a/bin/fm-bearings-snapshot.sh +++ b/bin/fm-bearings-snapshot.sh @@ -19,8 +19,9 @@ # explicitly (the prs: line and the omitted[] surfaces) what was not requested, so an # absence is never ambiguous. # -# This wrapper consumes canonical status decisions plus structured captain-held -# backlog items. It never infers decisions from report or visual-review prose. +# This wrapper consumes canonical status decisions plus canonically normalized +# backlog roles, unresolved blockers, and captain actionability. It never infers +# decisions from report or visual-review prose or reimplements snapshot semantics. # # Main-home inventory validity comes from the canonical snapshot's main_inventory # object (orphan structured in-flight without meta, unstructured current rows). @@ -111,9 +112,10 @@ landed merges this home's Done with registered secondmate homes' Done, bounded b with omitted[] disclosure. Default selection is balanced across deterministic home order while preserving each home's internal newest-first order; sparse homes do not waste capacity. --all-landed reveals the full global newest-first set. -For every registered secondmate, validated structured state from its own home is - authoritative. Parent events and bounded terminal reads are labeled fallback or - contradiction evidence and never become current work. +For every registered secondmate, readable structured facts from its own home are + authoritative, including independently trustworthy surfaces from a partial summary. + Parent events and bounded terminal reads are labeled fallback or contradiction + evidence and never become current work. Opt-in surfaces: --fields bodies|paths|actions|endpoints, --all-in-flight, --all-decisions, --all-secondmates, --all-landed, --all-reports, --all-queued, --all-recorded-prs, --all-unhealthy, --all-pr-repos, --include-prs (adds candidate_prs). @@ -326,6 +328,7 @@ MODEL=$(printf '%s' "$SNAP" | jq \ | (if $all_landed == 1 then $landed_sorted else ($per_home_groups | round_robin_landed($landed_n)) end) as $done | ($done | map(.id)) as $done_ids | ([.tasks[] | select(.kind != "secondmate") | .id]) as $live_ids + | ([.tasks[] | select(.kind != "secondmate" and .current_state.state == "working") | .id]) as $working_ids | ($live_ids + $done_ids) as $rel_ids | ([ .tasks[] | select(.endpoint.exists == false or .endpoint.agent_alive == "dead") @@ -366,8 +369,11 @@ MODEL=$(printf '%s' "$SNAP" | jq \ provenance:.provenance.selected,freshness:.freshness.status, age_seconds:.freshness.age_seconds,contradiction:(.contradiction // false), reason:(.current.reason // "-")} ]) as $secondmates_all - | ([ .tasks[] | select(.kind != "secondmate") | { - id, kind, + | ([ .tasks[] + | select(.kind != "secondmate") + | select(.backlog.current_role != "program") + | select(.backlog.current_role != "held" or .current_state.state == "working") + | {id, kind, state: .current_state.state, doing: ((.current_state.detail // "") as $d | (if $d != "" then $d else (.hints.last_event_text // "") end) | trunc(90)) @@ -377,8 +383,7 @@ MODEL=$(printf '%s' "$SNAP" | jq \ | {id,kind:"secondmate",state:.bearings_state, doing:([.active_children[] | .id + ": " + (.doing // .state)] | join("; ") | trunc(90))} ]) as $in_flight_all | ([ .backlog.records[] - | select(.state == "queued" and .structured and .kind == "captain" - and .hold_kind == "captain" and .hold_reason != null) + | select(.structured and .captain_actionable == true) | {id,key:.id,verb:"captain-hold", summary:((.title + ": " + .hold_reason) | trunc(90)),owner:"(main)"} ] + [ (.secondmate_current.records // [])[] as $m | $m.decisions_open[]? @@ -393,18 +398,23 @@ MODEL=$(printf '%s' "$SNAP" | jq \ owner:"(main)"}] else [] end) + [ .backlog.records[] - | select(.state == "queued" and .structured) - | select((.kind == "captain" and .hold_kind == "captain" and .hold_reason != null) | not) + | . as $record + | select(.structured and + (.state == "queued" or + (.state == "in_flight" and .current_role == "held" and ($working_ids | index($record.id) | not)))) + | select(.captain_actionable != true) | select(($all_queued == 1) or (((.body_excerpt // "") | test("SUPERSEDED|NOT REQUIRED|NOT-REQUIRED|DEFERRED"; "i")) | not)) - | {id, title:(.title | trunc(60)), blocked_by:(.blocked_by // "-"), - reason:((.blocked_reason // "-") | trunc(40)),owner:"(main)"} ] + | {id, title:(.title | trunc(60)), + blocked_by:((.unresolved_blocker_ids // []) | if length > 0 then join(",") else "-" end | trunc(120)), + reason:((.hold_reason // .blocked_reason // "-") | trunc(40)),owner:"(main)"} ] + [ (.secondmate_current.records // [])[] as $m | select($m.provenance.selected == "structured-home") | $m.queued[]? - | select((.kind == "captain" and .hold_kind == "captain" and .hold_reason != null) | not) - | {id,title:(.title | trunc(60)),blocked_by:(.blocked_by // "-"), - reason:((.blocked_reason // "-") | trunc(40)),owner:$m.id} ]) as $gates_all + | select(.captain_actionable != true) + | {id,title:(.title | trunc(60)), + blocked_by:((.unresolved_blocker_ids // []) | if length > 0 then join(",") else "-" end | trunc(120)), + reason:((.hold_reason // .blocked_reason // "-") | trunc(40)),owner:$m.id} ]) as $gates_all | ([ .scout_reports[] | . as $r | select(($all_reports == 1) or (($rel_ids | index($r.id)) != null)) diff --git a/bin/fm-fleet-snapshot.sh b/bin/fm-fleet-snapshot.sh index de6e0c5..1dee81b 100755 --- a/bin/fm-fleet-snapshot.sh +++ b/bin/fm-fleet-snapshot.sh @@ -16,7 +16,10 @@ # Canonical tasks-axi rows are structured; free-form non-empty lines in # those sections are preserved as unstructured records. # Structured rows preserve captain-hold metadata such as hold_kind and -# hold_reason when tasks-axi emits it. +# hold_reason when tasks-axi emits it. They also carry normalized current_role, +# requires_child_metadata, blocked_by_ids, unresolved_blocker_ids, and +# captain_actionable fields. Repeated blocker tokens remain ordered; a blocker +# resolves only when its structured record is Done, and missing ids stay open. # tasks[]: one row per state/.meta, sorted by id. # current_state is parsed from bin/fm-crew-state.sh and preserves # state, source, detail, and raw line separately. @@ -40,12 +43,14 @@ # for registered secondmates, selected from validated structured state inside # each home with explicit provenance, freshness, endpoint evidence, and unknown # failure reasons. Parent status and bounded terminal evidence are historical, -# untrusted supplements only and never override a valid structured summary. +# untrusted supplements only and never override readable structured-home facts. # Each structured-home record carries active_children, decisions_open, holds, -# queued, landed, endpoints, counts, and omitted; captain holds appear in -# decisions_open and are also preserved in queued with hold metadata. -# secondmate_landed: {records[],truncated[],unreadable[]} - the compatibility -# landed-work roll-up derived from secondmate_current. +# queued, landed, endpoints, counts, and omitted. Actionable captain holds +# appear in decisions_open; blocked captain holds remain queued with metadata. +# secondmate_landed: {records[],truncated[],unreadable[],partial[]} - the +# compatibility landed-work roll-up derived from secondmate_current. Readable +# structured homes with an unknown current classification are partial, not +# unreadable, and retain independently trustworthy structured surfaces. # secondmate_guidance: return-channel action note for renderers and bearings. # # Compatibility: JSON is the primary machine-readable surface. @@ -140,9 +145,11 @@ JSON is the stable machine-readable output contract. --secondmate-home-summary emits the bounded structured summary used after a validated registered-home handoff. It is local-only, skips nested secondmate -aggregation, and marks missing or unstructured current backlog state invalid. -Active tasks-axi captain holds appear as decisions_open and stay visible in -queued with hold_reason and hold_kind for downstream projections. +aggregation, and marks inventory contradictions or unavailable child state invalid. +Its invalidity object names the normalized failure kind and affected ids. +Actionable tasks-axi captain holds appear as decisions_open and stay visible in +queued with hold_reason, hold_kind, and plural blocker fields for downstream +projections. A captain hold is actionable only when every blocker is Done. Cross-home reads use FM_SNAPSHOT_SECONDMATES (default 20, 0 lifts the count bound), FM_SNAPSHOT_SECONDMATE_TIMEOUT, and FM_SNAPSHOT_SECONDMATE_MAX_BYTES. Terminal contradiction evidence uses @@ -285,6 +292,9 @@ backlog_json() { # [] - defaults to this home's $BACKLOG | sub("[[:space:]]*blocked-by:[[:space:]]+[^[:space:])]+[[:space:]]+-[[:space:]]+.*$"; "") | gsub("[[:space:]]*blocked-by:[[:space:]]+[^[:space:]]+"; "") | clean_title; + def blocked_by_ids($rest): + [ $rest | scan("blocked-by:[[:space:]]+(?[^[:space:])]+)") | .[0] ] + | reduce .[] as $id ([]; if index($id) == null then . + [$id] else . end); def blocked_reason($rest): cap($rest; ".*blocked-by:[[:space:]]*[^[:space:])]+[[:space:]]+-[[:space:]]*(?.*)$") as $reason | if $reason == null then null @@ -325,6 +335,7 @@ backlog_json() { # [] - defaults to this home's $BACKLOG hold_reason:metadata($rest; "hold"), hold_kind:metadata($rest; "hold-kind"), blocked_by:cap($rest; ".*blocked-by:[[:space:]]*(?[^[:space:])]+).*"), + blocked_by_ids:blocked_by_ids($rest), blocked_reason:blocked_reason($rest), since:metadata_word($rest; "since"), merged:metadata_word($rest; "merged"), @@ -360,6 +371,28 @@ backlog_json() { # [] - defaults to this home's $BACKLOG if (.body_lines | length) > 0 then .body_excerpt = ((.body_lines | join(" "))[:240]) else . end) + | .records as $records + | (reduce ($records[] | select(.structured)) as $record ({}; + .[$record.id] = ((.[$record.id] // true) and ($record.state == "done")))) as $resolved_ids + | .records |= map( + if .structured then + . as $record + | .unresolved_blocker_ids = [ + $record.blocked_by_ids[] as $blocker + | select($resolved_ids[$blocker] != true) + | $blocker + ] + | .current_role = + (if .state == "in_flight" and .hold_reason != null and .hold_kind != null then "held" + elif .state == "in_flight" and .kind == "program" then "program" + elif .state == "in_flight" then "worker" + elif .state == "queued" then "queued" + else "done" end) + | .requires_child_metadata = (.current_role == "worker") + | .captain_actionable = + (.state == "queued" and .kind == "captain" and .hold_kind == "captain" + and .hold_reason != null and (.unresolved_blocker_ids | length) == 0) + else . end) | del(.section,.order) ' < "$backlog" } @@ -536,7 +569,8 @@ main_inventory_json() { # --argjson tasks "$2" ' ([ $backlog.records[]? | select((.state == "in_flight" or .state == "queued") and (.structured | not)) ]) as $unstructured_current - | ([ $backlog.records[]? | select(.state == "in_flight" and .structured) ]) as $owned_in_flight + | ([ $backlog.records[]? + | select(.state == "in_flight" and .structured and .requires_child_metadata) ]) as $owned_in_flight | ([ $owned_in_flight[] | select(.id as $id | [$tasks[].id] | index($id) | not) | .id ]) as $orphan_in_flight @@ -573,9 +607,14 @@ secondmate_home_summary_json() { # ([ $backlog.records[]? | select((.state == "in_flight" or .state == "queued") and (.structured | not)) ]) as $unstructured_current | ([ $backlog.records[]? | select(.state == "in_flight" and .structured) ]) as $owned_in_flight - | ([ $backlog.records[]? | select(.state == "queued" and .structured) ]) as $queued_all + | ([ $backlog.records[]? + | select(.structured and + (.state == "queued" or + (.state == "in_flight" and .current_role == "held" + and (.id as $id + | any($tasks[]; .id == $id and .current_state.state == "working") | not)))) ]) as $queued_all | ([ $queued_all[] - | select(.kind == "captain" and .hold_kind == "captain" and .hold_reason != null) + | select(.captain_actionable == true) | {id,key:.id,verb:"captain-hold",summary:(.title | trunc(160)), reason:(.hold_reason | trunc(160)),source:"backlog"} ]) as $captain_holds_all | ([ $backlog.records[]? | select(.state == "done" and .structured and .kind != "captain") @@ -585,19 +624,38 @@ secondmate_home_summary_json() { # local_note:((.local_note // null) | if . == null then null else trunc(120) end),completion} ] | sort_by([(.completion.date // ""), .id]) | reverse) as $landed_all | ([ $tasks[] | select(.current_state.state == "unknown") ]) as $unknown_children - | ([ $owned_in_flight[] | select(.id as $id | [$tasks[].id] | index($id) | not) ]) as $orphan_in_flight + | ([ $owned_in_flight[] + | select(.requires_child_metadata) + | select(.id as $id | [$tasks[].id] | index($id) | not) ]) as $orphan_in_flight | ([ $tasks[] - | select(.current_state.state == "working" - or .current_state.state == "parked" - or .current_state.state == "paused" - or .current_state.state == "blocked") | select(.id as $id | [$owned_in_flight[].id] | index($id) | not) - | {id,state:.current_state.state} ]) as $unowned_current + | {id,state:.current_state.state} ]) as $unowned_children | ([ $owned_in_flight[] as $work | $tasks[] | select(.id == $work.id and (.current_state.state == "done" or .current_state.state == "failed")) | {id,state:.current_state.state} ]) as $terminal_in_flight + | ([if $backlog.present != true then + {kind:"missing_backlog",ids:[],reason:"missing structured backlog"} + else empty end, + if ($unstructured_current | length) > 0 then + {kind:"unstructured_current",ids:[],reason:"unstructured current backlog row"} + else empty end, + if ($orphan_in_flight | length) > 0 then + {kind:"orphan_in_flight",ids:($orphan_in_flight | map(.id)), + reason:("in-flight backlog item has no child metadata: " + ($orphan_in_flight | map(.id) | join(", ")))} + else empty end, + if ($unowned_children | length) > 0 then + {kind:"unowned_current",ids:($unowned_children | map(.id)), + reason:("live child state has no in-flight backlog item: " + + ($unowned_children | map(.id + "=" + .state) | join(", ")))} + else empty end, + if ($terminal_in_flight | length) > 0 then + {kind:"terminal_in_flight",ids:($terminal_in_flight | map(.id)), + reason:("in-flight backlog item has terminal child state: " + + ($terminal_in_flight | map(.id + "=" + .state) | join(", ")))} + else empty end]) as $strict_invalidities | ([ $owned_in_flight[] as $work + | select($work.current_role != "program") | $tasks[] | select(.id == $work.id and .current_state.state == "working") | {id,kind,state:.current_state.state,source:.current_state.source, @@ -605,30 +663,33 @@ secondmate_home_summary_json() { # | ($captain_holds_all + ([ $tasks[] as $t | ($t.hints.open_decisions // [])[] | {id:$t.id,key,verb,summary:(.summary | trunc(160)),reason:null,source:"status"} ])) as $decisions_all - | ([ $queued_all[] | select(.blocked_by != null) - | {id:(.id | trunc(120)),title:(.title | trunc(90)),blocked_by:(.blocked_by | trunc(120)),reason:((.blocked_reason // "blocked") | trunc(120)),source:"backlog"} ] + | ([ $queued_all[] + | select((.unresolved_blocker_ids | length) > 0 or (.hold_reason != null and .hold_kind != null)) + | {id:(.id | trunc(120)),title:(.title | trunc(90)), + blocked_by:((.unresolved_blocker_ids | join(",")) | if . == "" then null else trunc(120) end), + blocked_by_ids:(.blocked_by_ids | map(trunc(120))), + unresolved_blocker_ids:(.unresolved_blocker_ids | map(trunc(120))), + reason:((.hold_reason // .blocked_reason // "blocked") | trunc(120)),source:"backlog"} ] + [ $owned_in_flight[] as $work | $tasks[] | select(.id == $work.id and (.current_state.state == "parked" or .current_state.state == "paused" or .current_state.state == "blocked")) + | select(($work.hold_reason != null and $work.hold_kind != null) | not) | {id,title:((.backlog.title // .id) | trunc(90)),blocked_by:null, + blocked_by_ids:[],unresolved_blocker_ids:[], reason:((.current_state.detail // .current_state.state) | trunc(120)),source:"child-state"} ]) as $holds_all | ($backlog.present == true and ($unstructured_current | length) == 0 and ($unknown_children | length) == 0 and ($orphan_in_flight | length) == 0 - and ($unowned_current | length) == 0 + and ($unowned_children | length) == 0 and ($terminal_in_flight | length) == 0) as $valid - | (if $backlog.present != true then "missing structured backlog" - elif ($unstructured_current | length) > 0 then "unstructured current backlog row" - elif ($unknown_children | length) > 0 then "child current state unavailable" - elif ($orphan_in_flight | length) > 0 then "in-flight backlog item has no child metadata" - elif ($unowned_current | length) > 0 then - "live child state has no in-flight backlog item: " + - ($unowned_current | map(.id + "=" + .state) | join(", ")) - elif ($terminal_in_flight | length) > 0 then - "in-flight backlog item has terminal child state: " + - ($terminal_in_flight | map(.id + "=" + .state) | join(", ")) + | (if ($strict_invalidities | length) > 0 then $strict_invalidities[0].reason + elif ($unknown_children | length) > 0 then + "child current state unavailable: " + ($unknown_children | map(.id) | join(", ")) else null end) as $reason + | (if ($strict_invalidities | length) > 0 then $strict_invalidities[0] | del(.reason) + elif ($unknown_children | length) > 0 then {kind:"child_current_unavailable",ids:($unknown_children | map(.id))} + else {kind:null,ids:[]} end) as $invalidity | (if $valid | not then "unknown" elif any($decisions_all[]; .verb == "needs-decision" or .verb == "captain-hold") then "captain_decision" elif ($active_all | length) > 0 then "active_child_work" @@ -640,15 +701,19 @@ secondmate_home_summary_json() { # home:$home, valid:$valid, reason:$reason, + invalidity:$invalidity, state:$state, active_children:$active_all[:$child_n], decisions_open:$decisions_all[:$decisions_n], holds:$holds_all[:$queued_n], queued:([$queued_all[] | {id:(.id | trunc(120)),title:(.title | trunc(120)), blocked_by:((.blocked_by // null) | if . == null then null else trunc(120) end), + blocked_by_ids:((.blocked_by_ids // []) | map(trunc(120))), + unresolved_blocker_ids:((.unresolved_blocker_ids // []) | map(trunc(120))), blocked_reason:((.blocked_reason // null) | if . == null then null else trunc(160) end), hold_reason:((.hold_reason // null) | if . == null then null else trunc(160) end), hold_kind:((.hold_kind // null) | if . == null then null else trunc(40) end), + captain_actionable:(.captain_actionable // false), repo:((.repo // null) | if . == null then null else trunc(120) end), kind:((.kind // null) | if . == null then null else trunc(40) end)}][:$queued_n]), landed:(if $landed_n == 0 then $landed_all else $landed_all[:$landed_n] end), @@ -1008,7 +1073,7 @@ parent_evidence_reconciliation_json() { # local tasks=$1 registry union rows total_registered total shown truncated local row id home registered registry_error task status_file event_raw event_note event_epoch event_age - local activity_scan activities decisions reconciliation provenance freshness reason summary summary_rc summary_bytes state terminal terminal_contradiction contradiction + local activity_scan activities decisions reconciliation provenance freshness reason summary summary_rc summary_bytes summary_valid summary_reason summary_invalidity state current_reason terminal terminal_contradiction contradiction local records='[]' seen_homes='' registry=$(registry_secondmates_json) || return 1 union=$(jq -n --argjson registry "$registry" --argjson tasks "$tasks" ' @@ -1054,6 +1119,7 @@ secondmate_current_json() { # reason=$registry_error summary='{}' + summary_valid=false if [ -z "$reason" ] && [ -z "$home" ]; then reason="no recorded secondmate home"; fi if [ -z "$reason" ]; then case "$home" in @@ -1096,16 +1162,33 @@ secondmate_current_json() { # reason="structured home snapshot exceeded byte limit" elif ! printf '%s' "$summary" | jq -e --arg home "$home" --arg generated "$SNAPSHOT_NOW" ' .schema == "fm-secondmate-home-summary.v1" and .home == $home and .generated == $generated + and (.valid | type) == "boolean" and (.state | type) == "string" + and (.invalidity | type) == "object" and (.invalidity.ids | type) == "array" + and (.active_children | type) == "array" and (.decisions_open | type) == "array" + and (.holds | type) == "array" and (.queued | type) == "array" + and (.landed | type) == "array" and (.endpoints | type) == "array" + and (.counts | type) == "object" and (.omitted | type) == "array" ' >/dev/null 2>&1; then reason="structured home snapshot was malformed or stale" - elif [ "$(printf '%s' "$summary" | jq -r '.valid')" != true ]; then - reason="structured home state invalid: $(printf '%s' "$summary" | jq -r '.reason // "unknown reason"')" + else + summary_valid=$(printf '%s' "$summary" | jq -r '.valid') + if [ "$summary_valid" != true ]; then + summary_reason=$(printf '%s' "$summary" | jq -r '.reason // "unknown reason"') + summary_invalidity=$(printf '%s' "$summary" | jq -r '.invalidity.kind // "unknown"') + if [ "$summary_invalidity" != child_current_unavailable ]; then + reason="structured home state invalid: $summary_reason" + fi + fi fi fi fi if [ -z "$reason" ]; then state=$(printf '%s' "$summary" | jq -r '.state') + current_reason= + if [ "$summary_valid" != true ]; then + current_reason="structured home state invalid: $(printf '%s' "$summary" | jq -r '.reason // "unknown reason"')" + fi reconciliation=$(parent_evidence_reconciliation_json "$summary" "$activities" "$decisions") contradiction=$(printf '%s' "$reconciliation" | jq -r '.contradiction') terminal_contradiction=$(printf '%s' "$reconciliation" | jq -r --arg note "$event_note" ' @@ -1118,13 +1201,15 @@ secondmate_current_json() { # fi if printf '%s' "$terminal" | jq -e '.contradiction == true' >/dev/null; then contradiction=true; fi record=$(jq -n \ - --arg id "$id" --arg home "$home" --arg state "$state" --arg observed "$SNAPSHOT_NOW" \ - --argjson registered "$registered" --argjson summary "$summary" --argjson decisions "$decisions" \ + --arg id "$id" --arg home "$home" --arg state "$state" --arg current_reason "$current_reason" --arg observed "$SNAPSHOT_NOW" \ + --argjson registered "$registered" --argjson summary "$summary" --argjson summary_valid "$summary_valid" --argjson decisions "$decisions" \ --argjson activities "$activities" --argjson activity_scan "$activity_scan" \ --argjson reconciliation "$reconciliation" --argjson terminal "$terminal" --argjson contradiction "$contradiction" \ --arg event_raw "$event_raw" --arg event_note "$event_note" --argjson event_age "$event_age" ' - {id:$id,home:$home,registered:$registered,current:{state:$state,reason:null}, - provenance:{selected:"structured-home",structured_home:$home,parent_event_role:"historical-only"}, + {id:$id,home:$home,registered:$registered, + current:{state:$state,reason:($current_reason | if . == "" then null else . end)},invalidity:$summary.invalidity, + provenance:{selected:"structured-home",structured_home:$home,summary_valid:$summary_valid, + trust:(if $summary_valid then "complete" else "partial-structured" end),parent_event_role:"historical-only"}, freshness:{status:"fresh",observed_at:$observed,age_seconds:0}, active_children:$summary.active_children, decisions_open:$summary.decisions_open,holds:$summary.holds,queued:$summary.queued, @@ -1151,7 +1236,7 @@ secondmate_current_json() { # --argjson registered "$registered" --argjson event_age "$event_age" --argjson activities "$activities" --argjson activity_scan "$activity_scan" \ --argjson decisions "$decisions" --argjson terminal "$terminal" ' {id:$id,home:($home | if . == "" then null else . end),registered:$registered, - current:{state:"unknown",reason:$reason}, + current:{state:"unknown",reason:$reason},invalidity:null, provenance:{selected:$provenance,structured_home:($home | if . == "" then null else . end),parent_event_role:"fallback-only-not-current"}, freshness:{status:$freshness,observed_at:$observed,age_seconds:$event_age}, active_children:[],decisions_open:[],holds:[],queued:[],landed:[],endpoints:[],counts:{active_children:0,decisions_open:0,holds:0,queued:0,landed:0,endpoints:0},omitted:[], @@ -1182,8 +1267,11 @@ secondmate_landed_from_current_json() { # | select(.provenance.selected == "structured-home" and (.counts.landed > (.landed | length))) | .home], unreadable:[ $current.records[] - | select(.current.state == "unknown") - | .home // ("<" + .id + ": unavailable>")]} + | select(.current.state == "unknown" and .provenance.selected != "structured-home") + | .home // ("<" + .id + ": unavailable>")], + partial:[ $current.records[] + | select(.current.state == "unknown" and .provenance.selected == "structured-home") + | .home // ("<" + .id + ": partial>")]} | .records |= sort_by([(.completion.date // ""), .id]) | .records |= reverse' } diff --git a/bin/fm-install-shellcheck.sh b/bin/fm-install-shellcheck.sh index 1a21d32..45e1844 100755 --- a/bin/fm-install-shellcheck.sh +++ b/bin/fm-install-shellcheck.sh @@ -14,7 +14,17 @@ DESTINATION=${1:?usage: fm-install-shellcheck.sh } TMP=$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/fm-shellcheck.XXXXXX") trap 'rm -rf "$TMP"' EXIT -curl -fsSL "$URL" -o "$TMP/$ARCHIVE" +DOWNLOAD_ATTEMPTS=3 +download_attempt=1 +while ! curl -fsSL "$URL" -o "$TMP/$ARCHIVE"; do + [ "$download_attempt" -lt "$DOWNLOAD_ATTEMPTS" ] || { + printf 'fm-install-shellcheck.sh: download failed after %s attempts\n' "$DOWNLOAD_ATTEMPTS" >&2 + exit 1 + } + printf 'fm-install-shellcheck.sh: download attempt %s failed; retrying\n' "$download_attempt" >&2 + sleep "$download_attempt" + download_attempt=$((download_attempt + 1)) +done ACTUAL_SHA256=$(sha256sum "$TMP/$ARCHIVE" | awk '{print $1}') [ "$ACTUAL_SHA256" = "$SHA256" ] || { printf 'fm-install-shellcheck.sh: checksum mismatch for %s\n' "$ARCHIVE" >&2 diff --git a/docs/architecture.md b/docs/architecture.md index e24b8d0..4c54ae3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -42,7 +42,8 @@ A registered secondmate's validated home is the authority for bearings current s The original cross-home projection instead treated the secondmate agent as an ordinary parent task, so an idle secondmate's `fm-crew-state` fallback selected the latest append-only parent status event even when structured state in the registered home contradicted it. The parent-status contract also required explicit keyed resolution for decisions and blockers but not for a material `working` phase, so a start event could remain unsuperseded after the corresponding home backlog had moved the work to Done. Generated secondmate charters now key material routed-work phases and close them with a same-key later state or `resolved` event, while the structured home remains authoritative even if that closure is missing. -Cross-home reads validate the seeded identity and operational-directory boundaries, use per-home time and output bounds, and classify unavailable, invalid, malformed, or inconsistent structured state as unknown rather than reviving a parent event as current work. +Cross-home reads validate the seeded identity and operational-directory boundaries, use per-home time and output bounds, and classify unavailable, malformed, or inconsistent structured state as unknown rather than reviving a parent event as current work. +When only an owned child's current classification is unavailable, the home classification stays unknown while independently trustworthy structured decisions, holds, queued and landed records, endpoint identities, counts, and provenance remain available; every other invalid path stays strict and exposes none of those child-derived surfaces. A bounded direct-report terminal tail can help diagnose a mismatch by showing that historical parent wording is still visible, but it is untrusted supplemental evidence because scrollback, prompts, copied output, idle shells, and agent prose are not durable state. The snapshot strips control sequences, retains only capture metadata and literal event-corroboration flags, and never lets terminal evidence override a valid structured classification. The default path remains local-only; live GitHub enrichment exists only behind the bearings `--include-prs` opt-in. diff --git a/docs/decision-hold-lifecycle.md b/docs/decision-hold-lifecycle.md index e29fe1f..234055a 100644 --- a/docs/decision-hold-lifecycle.md +++ b/docs/decision-hold-lifecycle.md @@ -31,9 +31,10 @@ A failed intermediate step leaves the hold open. ## Structured read surfaces `bin/fm-fleet-snapshot.sh` parses canonical tasks-axi `(hold: ...)` and `(hold-kind: captain)` metadata alongside existing backlog fields. -Its secondmate-home summary classifies an active captain hold as `captain_decision` and preserves the owning home. +It resolves every repeated `blocked-by:` edge against structured Done records, keeps missing blockers unresolved, and classifies only an unblocked captain hold as actionable. +Its secondmate-home summary classifies an actionable captain hold as `captain_decision` and preserves blocked captain holds as queued work in the owning home. -`bin/fm-bearings-snapshot.sh` projects active captain holds into `decisions_open` and excludes them from ordinary queued gates. +`bin/fm-bearings-snapshot.sh` projects actionable captain holds into `decisions_open` and leaves blocked captain holds in ordinary queued gates. It excludes completed kind `captain` records from Recently Landed. The projection remains read-only and does not inspect historical prose. @@ -41,6 +42,7 @@ The projection remains read-only and does not inspect historical prose. Verification date: 2026-07-14. Additional quoted `blocked_by` regression verification date: 2026-07-17. +Plural blocker-readiness and mixed-home projection verification date: 2026-07-22. The focused end-to-end regression uses only synthetic `sample` identities and decision text. It begins with a completed investigation and visual review whose genuine unresolved choice exists only in the report. @@ -62,12 +64,15 @@ ok - main-home and secondmate-home captain holds remain correctly routed ok - resolve matches first/middle/last in quoted blocked_by and rejects a genuinely absent id $ bash tests/fm-fleet-snapshot-view.test.sh +ok - backlog normalization preserves strict roles and resolves every blocker compatibly ok - durable captain-held transfer closes the duplicate live status decision ok - snapshot parses tasks-axi rows and respects operational overrides $ bash tests/fm-bearings-snapshot.test.sh ok - a completed scout with decision-like report prose is a pointer, not pending ok - action-free items (working/done/queued/landed) do not leak into Captain's Call +ok - mixed secondmate roles, partial state, and captain readiness project independently +ok - main and secondmate captain actionability use the same blocker readiness $ bash tests/fm-brief.test.sh ok - fm-brief.sh: investigation and visual-review completions load the shared decision policy diff --git a/tests/fm-bearings-snapshot.test.sh b/tests/fm-bearings-snapshot.test.sh index b811be3..f8afefa 100755 --- a/tests/fm-bearings-snapshot.test.sh +++ b/tests/fm-bearings-snapshot.test.sh @@ -474,6 +474,40 @@ test_bad_secondmate_homes_never_revive_parent_work() { pass "missing, invalid, unreadable, malformed, and timed-out homes stay explicit unknowns" } +test_oversized_secondmate_summary_stays_strict_unknown() { + local home mate fakebin json i + home=$(make_home oversized-home) + mate="$TMP_ROOT/oversized-secondmate-home" + make_valid_secondmate_home oversized "$mate" + append_secondmate_registry "$home" oversized "$mate" + fm_write_secondmate_meta "$home/state/oversized.meta" "$mate" "firstmate:fm-oversized" sample + printf 'working [key=old]: stale parent activity\n' > "$home/state/oversized.status" + cat > "$mate/data/backlog.md" <<'EOF' +## In flight + +## Queued + +## Done +EOF + i=1 + while [ "$i" -le 30 ]; do + printf -- '- [x] landed-%02d - Bounded landed fixture %02d (repo: sample) (kind: ship) (done 2026-07-01)\n' \ + "$i" "$i" >> "$mate/data/backlog.md" + i=$((i + 1)) + done + fakebin=$(make_fakebin "$home") + json=$(FM_SNAPSHOT_SECONDMATE_MAX_BYTES=512 run "$home" "$fakebin" --json) + printf '%s' "$json" | jq -e ' + (.secondmates | any(.id == "oversized" and .state == "unknown" + and .provenance == "parent-event-fallback" + and (.reason | contains("exceeded byte limit")))) + and (.in_flight | any(.id == "oversized") | not) + and (.decisions_open | any(.owner == "oversized") | not) + and (.landed | any(.owner == "oversized") | not) + ' >/dev/null || fail "oversized summary revived or retained unvalidated surfaces: $json" + pass "an oversized secondmate summary retains the strict empty unknown fallback" +} + test_secondmate_and_child_bounds_are_disclosed() { local home fakebin id mate child json expanded canonical i home=$(make_home secondmate-bounds) @@ -1500,6 +1534,333 @@ EOF pass "counterfactual meta clears main inventory warning and projects the live task" } +test_mixed_secondmate_roles_partial_state_and_captain_readiness() { + local home fakebin hibit wheel sshhip ha canonical json + home=$(make_home mixed-domain-regressions) + : > "$home/data/secondmates.md" + hibit="$TMP_ROOT/mixed-hibit-home" + wheel="$TMP_ROOT/mixed-wheel-home" + sshhip="$TMP_ROOT/mixed-sshhip-home" + ha="$TMP_ROOT/mixed-ha-home" + make_valid_secondmate_home hibit "$hibit" + make_valid_secondmate_home wheel "$wheel" + make_valid_secondmate_home sshhip "$sshhip" + make_valid_secondmate_home home-assistant "$ha" + append_secondmate_registry "$home" hibit "$hibit" + append_secondmate_registry "$home" wheel "$wheel" + append_secondmate_registry "$home" sshhip "$sshhip" + append_secondmate_registry "$home" home-assistant "$ha" + + mkdir -p "$hibit/projects/worker" "$wheel/projects/worker" "$sshhip/projects/child" "$ha/projects/prep" + cat > "$hibit/data/backlog.md" <<'EOF' +## In flight +- [ ] dogfood-program - Long-lived dogfood program (repo: hibit) (kind: program) +- [ ] hibit-worker - Finalize progress (repo: hibit) (kind: ship) + +## Queued + +## Done +EOF + fm_write_meta "$hibit/state/hibit-worker.meta" \ + "window=firstmate:fm-hibit-worker" "worktree=$hibit/projects/worker" "project=hibit" \ + "harness=codex" "kind=ship" "mode=no-mistakes" + printf 'working: finalizing progress\n' > "$hibit/state/hibit-worker.status" + + cat > "$wheel/data/backlog.md" <<'EOF' +## In flight +- [ ] production-observation - Observe production (repo: wheelhouse) (kind: scout) (hold: documented no live worker) (hold-kind: external) +- [ ] wheel-worker - Initial triage (repo: wheelhouse) (kind: ship) + +## Queued + +## Done +EOF + fm_write_meta "$wheel/state/wheel-worker.meta" \ + "window=firstmate:fm-wheel-worker" "worktree=$wheel/projects/worker" "project=wheelhouse" \ + "harness=codex" "kind=ship" "mode=no-mistakes" + printf 'working: active validation\n' > "$wheel/state/wheel-worker.status" + + cat > "$sshhip/data/backlog.md" <<'EOF' +## In flight +- [ ] unreadable-child - Submit App Store build (repo: sshhip) (kind: ship) + +## Queued +- [ ] reviewer-decision - Choose reviewer remediation (repo: sshhip) (kind: captain) (hold: choose reviewer remediation A or B) (hold-kind: captain) + +## Done +- [x] prior-release - Prior release (repo: sshhip) (kind: ship) (done 2026-07-21) +EOF + fm_write_meta "$sshhip/state/unreadable-child.meta" \ + "window=firstmate:dead-sshhip-child" "worktree=$sshhip/projects/child" "project=sshhip" \ + "harness=codex" "kind=ship" "mode=no-mistakes" + + cat > "$ha/data/backlog.md" <<'EOF' +## In flight +- [ ] prep - Prepare canary (repo: home-assistant) (kind: ship) + +## Queued +- [ ] security - Security review (repo: home-assistant) (kind: ship) +- [ ] captain-run - Run captain canary blocked-by: prep blocked-by: security (repo: home-assistant) (kind: captain) (hold: captain runs canary) (hold-kind: captain) + +## Done +EOF + fm_write_meta "$ha/state/prep.meta" \ + "window=firstmate:fm-prep" "worktree=$ha/projects/prep" "project=home-assistant" \ + "harness=codex" "kind=ship" "mode=no-mistakes" + printf 'working: preparing canary\n' > "$ha/state/prep.status" + + fakebin=$(make_fakebin "$home") + canonical=$(PATH="$fakebin:$PATH" FM_HOME="$home" FM_SNAPSHOT_NOW=2026-07-11T18:00:00Z \ + "$ROOT/bin/fm-fleet-snapshot.sh" --json) + printf '%s' "$canonical" | jq -e ' + (.secondmate_current.records[] | select(.id == "hibit") + | .current.state == "active_child_work" + and [.active_children[].id] == ["hibit-worker"] + and ([.endpoints[].id] | index("dogfood-program") | not)) + and (.secondmate_current.records[] | select(.id == "wheel") + | .current.state == "active_child_work" + and [.active_children[].id] == ["wheel-worker"] + and [.queued[].id] == ["production-observation"] + and [.holds[].id] == ["production-observation"]) + and (.secondmate_current.records[] | select(.id == "sshhip") + | .current.state == "unknown" + and (.current.reason | contains("child current state unavailable: unreadable-child")) + and .provenance.selected == "structured-home" + and .provenance.summary_valid == false + and .provenance.trust == "partial-structured" + and .invalidity == {kind:"child_current_unavailable",ids:["unreadable-child"]} + and [.decisions_open[].id] == ["reviewer-decision"] + and [.holds[].id] == ["reviewer-decision"] + and [.queued[].id] == ["reviewer-decision"] + and [.landed[].id] == ["prior-release"] + and [.endpoints[].id] == ["unreadable-child"] + and .counts.decisions_open == 1 + and .counts.holds == 1 + and .counts.queued == 1 + and .counts.landed == 1 + and .counts.endpoints == 1) + and (.secondmate_landed.partial | length) == 1 + and (.secondmate_landed.partial[0] | endswith("/mixed-sshhip-home")) + and (.secondmate_landed.unreadable | length) == 0 + and (.secondmate_current.records[] | select(.id == "home-assistant") + | .current.state == "active_child_work" + and .decisions_open == [] + and [.active_children[].id] == ["prep"] + and (.queued[] | select(.id == "captain-run") + | .blocked_by_ids == ["prep", "security"] + and .unresolved_blocker_ids == ["prep", "security"] + and .captain_actionable == false)) + ' >/dev/null || fail "canonical mixed-domain classification was wrong: $canonical" + json=$(run "$home" "$fakebin" --json --fields bodies --all-landed) + printf '%s' "$json" | jq -e ' + ([.in_flight[].id] | sort) == ["hibit", "home-assistant", "wheel"] + and (.decisions_open | any(.id == "sshhip/reviewer-decision")) + and (.decisions_open | any(.id == "home-assistant/captain-run") | not) + and (.gates | any(.id == "production-observation" and .owner == "wheel" + and .reason == "documented no live worker")) + and (.gates | any(.id == "captain-run" and .owner == "home-assistant" + and .blocked_by == "prep,security")) + and (.secondmates | any(.id == "sshhip" and .state == "unknown" + and (.reason | contains("unreadable-child")))) + ' >/dev/null || fail "end-to-end mixed-domain projection was wrong: $json" + + sed '/unreadable-child/a\ +- [ ] ordinary-orphan - Unowned release task (repo: sshhip) (kind: ship)' \ + "$sshhip/data/backlog.md" > "$sshhip/data/backlog.next" + mv "$sshhip/data/backlog.next" "$sshhip/data/backlog.md" + canonical=$(PATH="$fakebin:$PATH" FM_HOME="$home" FM_SNAPSHOT_NOW=2026-07-11T18:00:00Z \ + "$ROOT/bin/fm-fleet-snapshot.sh" --json) + printf '%s' "$canonical" | jq -e ' + .secondmate_current.records[] | select(.id == "sshhip") + | .current.state == "unknown" + and (.current.reason | contains("in-flight backlog item has no child metadata: ordinary-orphan")) + and .provenance.selected != "structured-home" + and .invalidity == null + and .active_children == [] + and .decisions_open == [] + and .holds == [] + and .queued == [] + and .landed == [] + and .endpoints == [] + ' >/dev/null || fail "an unknown child masked a simultaneous ordinary orphan: $canonical" + sed '/ordinary-orphan/d' "$sshhip/data/backlog.md" > "$sshhip/data/backlog.next" + mv "$sshhip/data/backlog.next" "$sshhip/data/backlog.md" + + sed '/unreadable-child/d' "$sshhip/data/backlog.md" > "$sshhip/data/backlog.next" + mv "$sshhip/data/backlog.next" "$sshhip/data/backlog.md" + canonical=$(PATH="$fakebin:$PATH" FM_HOME="$home" FM_SNAPSHOT_NOW=2026-07-11T18:00:00Z \ + "$ROOT/bin/fm-fleet-snapshot.sh" --json) + printf '%s' "$canonical" | jq -e ' + .secondmate_current.records[] | select(.id == "sshhip") + | .current.state == "unknown" + and (.current.reason | contains("live child state has no in-flight backlog item: unreadable-child=unknown")) + and .provenance.selected != "structured-home" + and .invalidity == null + and .active_children == [] + and .decisions_open == [] + and .holds == [] + and .queued == [] + and .landed == [] + and .endpoints == [] + ' >/dev/null || fail "an unowned unknown child received partial structured projection: $canonical" + sed '/## In flight/a\ +- [ ] unreadable-child - Submit App Store build (repo: sshhip) (kind: ship)' \ + "$sshhip/data/backlog.md" > "$sshhip/data/backlog.next" + mv "$sshhip/data/backlog.next" "$sshhip/data/backlog.md" + + fm_write_meta "$wheel/state/production-observation.meta" \ + "window=firstmate:fm-production-observation" "worktree=$wheel/projects/worker" "project=wheelhouse" \ + "harness=codex" "kind=scout" "mode=scout" + printf 'paused: observation is deliberately held\n' > "$wheel/state/production-observation.status" + canonical=$(PATH="$fakebin:$PATH" FM_HOME="$home" FM_SNAPSHOT_NOW=2026-07-11T18:00:00Z \ + "$ROOT/bin/fm-fleet-snapshot.sh" --json) + printf '%s' "$canonical" | jq -e ' + .secondmate_current.records[] | select(.id == "wheel") + | ([.queued[] | select(.id == "production-observation")] | length) == 1 + and ([.holds[] | select(.id == "production-observation")] | length) == 1 + and ([.endpoints[] | select(.id == "production-observation")] | length) == 1 + ' >/dev/null || fail "held metadata plus a real child duplicated or discarded the record: $canonical" + + fm_write_meta "$sshhip/state/unreadable-child.meta" \ + "window=firstmate:fm-unreadable-child" "worktree=$sshhip/projects/child" "project=sshhip" \ + "harness=codex" "kind=ship" "mode=no-mistakes" + printf 'working: app store submission restored\n' > "$sshhip/state/unreadable-child.status" + json=$(run "$home" "$fakebin" --json) + printf '%s' "$json" | jq -e ' + (.secondmates | any(.id == "sshhip" and .state == "captain_decision" and .reason == "-")) + and ([.decisions_open[] | select(.id == "sshhip/reviewer-decision")] | length) == 1 + ' >/dev/null || fail "restoring the SSHHIP child did not clear only its narrow warning: $json" + + cat > "$ha/data/backlog.md" <<'EOF' +## In flight + +## Queued +- [ ] security - Security review (repo: home-assistant) (kind: ship) +- [ ] captain-run - Run captain canary blocked-by: prep blocked-by: security (repo: home-assistant) (kind: captain) (hold: captain runs canary) (hold-kind: captain) + +## Done +- [x] prep - Prepare canary (repo: home-assistant) (kind: ship) (done 2026-07-22) +EOF + rm "$ha/state/prep.meta" "$ha/state/prep.status" + json=$(run "$home" "$fakebin" --json) + printf '%s' "$json" | jq -e ' + (.decisions_open | any(.id == "home-assistant/captain-run") | not) + and (.gates | any(.id == "captain-run" and .owner == "home-assistant" and .blocked_by == "security")) + ' >/dev/null || fail "one remaining Home Assistant blocker became actionable: $json" + + cat > "$ha/data/backlog.md" <<'EOF' +## In flight + +## Queued +- [ ] captain-run - Run captain canary blocked-by: prep blocked-by: security (repo: home-assistant) (kind: captain) (hold: captain runs canary) (hold-kind: captain) + +## Done +- [x] prep - Prepare canary (repo: home-assistant) (kind: ship) (done 2026-07-22) +- [x] security - Security review (repo: home-assistant) (kind: ship) (done 2026-07-22) +EOF + json=$(run "$home" "$fakebin" --json) + printf '%s' "$json" | jq -e ' + ([.decisions_open[] | select(.id == "home-assistant/captain-run")] | length) == 1 + and (.gates | any(.id == "captain-run" and .owner == "home-assistant") | not) + ' >/dev/null || fail "zero Home Assistant blockers did not yield exactly one captain action: $json" + + sed 's/blocked-by: security/blocked-by: missing/' "$ha/data/backlog.md" > "$ha/data/backlog.next" + mv "$ha/data/backlog.next" "$ha/data/backlog.md" + json=$(run "$home" "$fakebin" --json) + printf '%s' "$json" | jq -e ' + (.decisions_open | any(.id == "home-assistant/captain-run") | not) + and (.gates | any(.id == "captain-run" and .owner == "home-assistant" and .blocked_by == "missing")) + ' >/dev/null || fail "a missing Home Assistant blocker was treated as Done: $json" + + sed 's/(kind: program)/(kind: mystery)/' "$hibit/data/backlog.md" > "$hibit/data/backlog.next" + mv "$hibit/data/backlog.next" "$hibit/data/backlog.md" + canonical=$(PATH="$fakebin:$PATH" FM_HOME="$home" FM_SNAPSHOT_NOW=2026-07-11T18:00:00Z \ + "$ROOT/bin/fm-fleet-snapshot.sh" --json) + printf '%s' "$canonical" | jq -e ' + .secondmate_current.records[] | select(.id == "hibit") + | .current.state == "unknown" + and (.current.reason | contains("in-flight backlog item has no child metadata: dogfood-program")) + and .provenance.selected != "structured-home" + and .active_children == [] + and .decisions_open == [] + and .holds == [] + and .queued == [] + and .landed == [] + and .endpoints == [] + ' >/dev/null || fail "an unrecognized worker kind no longer stayed strict: $canonical" + pass "mixed secondmate roles, partial state, and captain readiness project independently" +} + +test_main_captain_readiness_matches_secondmate_projection() { + local home fakebin json + home=$(make_home main-captain-readiness) + : > "$home/data/secondmates.md" + mkdir -p "$home/projects/prep" "$home/projects/observation" + cat > "$home/data/backlog.md" <<'EOF' +## In flight +- [ ] observation - Held observation (repo: firstmate) (kind: scout) (hold: watch production) (hold-kind: external) +- [ ] prep - Prepare canary (repo: firstmate) (kind: ship) + +## Queued +- [ ] review - Security review (repo: firstmate) (kind: ship) +- [ ] captain-run - Run captain canary blocked-by: prep blocked-by: review (repo: firstmate) (kind: captain) (hold: captain runs canary) (hold-kind: captain) + +## Done +EOF + fm_write_meta "$home/state/prep.meta" \ + "window=firstmate:fm-prep" "worktree=$home/projects/prep" "project=firstmate" \ + "harness=codex" "kind=ship" "mode=no-mistakes" + printf 'working: preparing main canary\n' > "$home/state/prep.status" + fm_write_meta "$home/state/observation.meta" \ + "window=firstmate:fm-observation" "worktree=$home/projects/observation" "project=firstmate" \ + "harness=codex" "kind=scout" "mode=scout" + printf 'paused: observation is deliberately held\n' > "$home/state/observation.status" + fakebin=$(make_fakebin "$home") + json=$(run "$home" "$fakebin" --json) + printf '%s' "$json" | jq -e ' + (.in_flight | any(.id == "prep")) + and (.in_flight | any(.id == "observation") | not) + and ([.gates[] | select(.id == "observation" and .reason == "watch production")] | length) == 1 + and (.decisions_open | any(.id == "captain-run") | not) + and (.gates | any(.id == "captain-run" and .blocked_by == "prep,review")) + ' >/dev/null || fail "main blocked captain action or held-child projection was wrong: $json" + + cat > "$home/data/backlog.md" <<'EOF' +## In flight + +## Queued +- [ ] review - Security review (repo: firstmate) (kind: ship) +- [ ] captain-run - Run captain canary blocked-by: prep blocked-by: review (repo: firstmate) (kind: captain) (hold: captain runs canary) (hold-kind: captain) + +## Done +- [x] prep - Prepare canary (repo: firstmate) (kind: ship) (done 2026-07-22) +EOF + rm "$home/state/prep.meta" "$home/state/prep.status" \ + "$home/state/observation.meta" "$home/state/observation.status" + json=$(run "$home" "$fakebin" --json) + printf '%s' "$json" | jq -e ' + (.decisions_open | any(.id == "captain-run") | not) + and (.gates | any(.id == "captain-run" and .blocked_by == "review")) + ' >/dev/null || fail "main one-blocker captain action became premature: $json" + + cat > "$home/data/backlog.md" <<'EOF' +## In flight + +## Queued +- [ ] captain-run - Run captain canary blocked-by: prep blocked-by: review (repo: firstmate) (kind: captain) (hold: captain runs canary) (hold-kind: captain) + +## Done +- [x] prep - Prepare canary (repo: firstmate) (kind: ship) (done 2026-07-22) +- [x] review - Security review (repo: firstmate) (kind: ship) (done 2026-07-22) +EOF + json=$(run "$home" "$fakebin" --json) + printf '%s' "$json" | jq -e ' + ([.decisions_open[] | select(.id == "captain-run")] | length) == 1 + and (.gates | any(.id == "captain-run") | not) + ' >/dev/null || fail "main zero-blocker captain action was not projected exactly once: $json" + pass "main and secondmate captain actionability use the same blocker readiness" +} + # The /bearings skill is the one owner of the four-section chat-response contract. # Assert it states exactly the four fixed sections in order, each with its explicit # empty-state sentence, documents the At Anchor exclusion, and mandates a chat that is @@ -1535,6 +1896,7 @@ test_parent_activity_evidence_is_bounded_and_disclosed test_active_child_overrides_old_parent_event test_structured_child_decision_reaches_captains_call test_bad_secondmate_homes_never_revive_parent_work +test_oversized_secondmate_summary_stays_strict_unknown test_secondmate_and_child_bounds_are_disclosed test_parent_decision_is_untrusted_contradiction_only test_parent_evidence_reconciles_by_verb_and_key @@ -1556,6 +1918,8 @@ test_captains_call_anti_leak test_main_orphan_in_flight_is_disclosed_not_invented test_main_unstructured_current_is_disclosed_with_structured_sibling test_main_orphan_counterfactual_meta_clears_inventory_warning +test_mixed_secondmate_roles_partial_state_and_captain_readiness +test_main_captain_readiness_matches_secondmate_projection test_chat_contract_four_sections test_completed_scout_report_not_pending test_open_decision_surfaces_end_to_end diff --git a/tests/fm-fleet-snapshot-view.test.sh b/tests/fm-fleet-snapshot-view.test.sh index c03e366..3c0288c 100755 --- a/tests/fm-fleet-snapshot-view.test.sh +++ b/tests/fm-fleet-snapshot-view.test.sh @@ -246,6 +246,99 @@ EOF pass "main_inventory discloses orphan/unstructured and clears when inventory is consistent" } +test_normalized_roles_and_plural_blocker_readiness() { + local home fakebin out + home=$(make_home normalized-records) + mkdir -p "$home/projects/worker" + cat > "$home/data/backlog.md" <<'EOF' +## In flight +- [ ] program - Aggregate program (repo: alpha) (kind: program) +- [ ] observation - Held observation (repo: alpha) (kind: scout) (hold: watch production) (hold-kind: external) +- [ ] worker - Real worker (repo: alpha) (kind: ship) +- [ ] orphan - Ordinary missing worker (repo: alpha) (kind: ship) + +## Queued +- [ ] review - Security review (repo: alpha) (kind: ship) +- [ ] captain-run - Run canary blocked-by: worker blocked-by: review (repo: alpha) (kind: captain) (hold: captain runs canary) (hold-kind: captain) + +## Done +EOF + fm_write_meta "$home/state/worker.meta" \ + "window=firstmate:fm-worker" "worktree=$home/projects/worker" "project=alpha" \ + "harness=codex" "kind=ship" "mode=ship" + printf 'working: preparing canary\n' > "$home/state/worker.status" + fakebin=$(make_fakebin "$home") + out=$(PATH="$fakebin:$PATH" FM_HOME="$home" "$SNAPSHOT" --json) + printf '%s' "$out" | jq -e ' + .main_inventory.orphan_in_flight == ["orphan"] + and (.backlog.records[] | select(.id == "program") + | .current_role == "program" and .requires_child_metadata == false) + and (.backlog.records[] | select(.id == "observation") + | .current_role == "held" and .requires_child_metadata == false) + and (.backlog.records[] | select(.id == "orphan") + | .current_role == "worker" and .requires_child_metadata == true) + and (.backlog.records[] | select(.id == "captain-run") + | .blocked_by == "review" + and .blocked_by_ids == ["worker", "review"] + and .unresolved_blocker_ids == ["worker", "review"] + and .captain_actionable == false) + ' >/dev/null || fail "normalized role or plural blocker fields were wrong: $out" + + cat > "$home/data/backlog.md" <<'EOF' +## In flight +- [ ] program - Aggregate program (repo: alpha) (kind: program) +- [ ] observation - Held observation (repo: alpha) (kind: scout) (hold: watch production) (hold-kind: external) + +## Queued +- [ ] review - Security review (repo: alpha) (kind: ship) +- [ ] captain-run - Run canary blocked-by: worker blocked-by: review (repo: alpha) (kind: captain) (hold: captain runs canary) (hold-kind: captain) + +## Done +- [x] worker - Real worker (repo: alpha) (kind: ship) (done 2026-07-22) +EOF + rm "$home/state/worker.meta" "$home/state/worker.status" + out=$(PATH="$fakebin:$PATH" FM_HOME="$home" "$SNAPSHOT" --json) + printf '%s' "$out" | jq -e ' + .backlog.records[] | select(.id == "captain-run") + | .blocked_by == "review" + and .blocked_by_ids == ["worker", "review"] + and .unresolved_blocker_ids == ["review"] + and .captain_actionable == false + ' >/dev/null || fail "one completed blocker did not leave exactly one unresolved id: $out" + + cat > "$home/data/backlog.md" <<'EOF' +## In flight +- [ ] program - Aggregate program (repo: alpha) (kind: program) +- [ ] observation - Held observation (repo: alpha) (kind: scout) (hold: watch production) (hold-kind: external) + +## Queued +- [ ] captain-run - Run canary blocked-by: worker blocked-by: review (repo: alpha) (kind: captain) (hold: captain runs canary) (hold-kind: captain) + +## Done +- [x] worker - Real worker (repo: alpha) (kind: ship) (done 2026-07-22) +- [x] review - Security review (repo: alpha) (kind: ship) (done 2026-07-22) +EOF + out=$(PATH="$fakebin:$PATH" FM_HOME="$home" "$SNAPSHOT" --json) + printf '%s' "$out" | jq -e ' + .backlog.records[] | select(.id == "captain-run") + | .blocked_by == "review" + and .blocked_by_ids == ["worker", "review"] + and .unresolved_blocker_ids == [] + and .captain_actionable == true + ' >/dev/null || fail "completed blockers did not make the captain hold actionable: $out" + + sed 's/blocked-by: review/blocked-by: missing/' "$home/data/backlog.md" > "$home/data/backlog.next" + mv "$home/data/backlog.next" "$home/data/backlog.md" + out=$(PATH="$fakebin:$PATH" FM_HOME="$home" "$SNAPSHOT" --json) + printf '%s' "$out" | jq -e ' + .backlog.records[] | select(.id == "captain-run") + | .blocked_by_ids == ["worker", "missing"] + and .unresolved_blocker_ids == ["missing"] + and .captain_actionable == false + ' >/dev/null || fail "a missing blocker was incorrectly treated as resolved: $out" + pass "backlog normalization preserves strict roles and resolves every blocker compatibly" +} + test_event_hints_follow_reconciled_current_state() { local home fakebin out home=$(make_home event-hints) @@ -662,6 +755,7 @@ test_parked_scout_decision_stays_pending() { test_empty_fleet_json test_fixture_snapshot_json test_main_inventory_orphan_and_unstructured_disclosure +test_normalized_roles_and_plural_blocker_readiness test_event_hints_follow_reconciled_current_state test_open_decision_survives_later_unrelated_event test_secondmate_open_decision_survives_live_endpoint diff --git a/tests/fm-lint.test.sh b/tests/fm-lint.test.sh index 5ca9d9e..9ced0a6 100755 --- a/tests/fm-lint.test.sh +++ b/tests/fm-lint.test.sh @@ -80,6 +80,62 @@ test_ci_installs_and_logs_the_pinned_version() { pass "CI installs and logs the pinned ShellCheck version from the one owner" } +test_installer_retries_transient_download_failure() { + local tmp fakebin destination out + tmp=$(fm_test_tmproot fm-shellcheck-download) + fakebin=$(fm_fakebin "$tmp") + destination="$tmp/bin" + + cat > "$fakebin/curl" <<'SH' +#!/usr/bin/env bash +count=0 +[ ! -f "$CURL_COUNT" ] || count=$(cat "$CURL_COUNT") +count=$((count + 1)) +printf '%s\n' "$count" > "$CURL_COUNT" +[ "$count" -gt 1 ] || exit 35 +while [ "$#" -gt 0 ]; do + if [ "$1" = "-o" ]; then + : > "$2" + exit 0 + fi + shift +done +exit 2 +SH + cat > "$fakebin/sha256sum" <<'SH' +#!/usr/bin/env bash +printf '8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198 %s\n' "$1" +SH + cat > "$fakebin/tar" <<'SH' +#!/usr/bin/env bash +while [ "$#" -gt 0 ]; do + if [ "$1" = "-C" ]; then + mkdir -p "$2/shellcheck-v0.11.0" + cat > "$2/shellcheck-v0.11.0/shellcheck" <<'EOF' +#!/usr/bin/env bash +printf 'ShellCheck - shell script analysis tool\nversion: 0.11.0\n' +EOF + chmod +x "$2/shellcheck-v0.11.0/shellcheck" + exit 0 + fi + shift +done +exit 2 +SH + cat > "$fakebin/sleep" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + chmod +x "$fakebin/curl" "$fakebin/sha256sum" "$fakebin/tar" "$fakebin/sleep" + + out=$(CURL_COUNT="$tmp/curl-count" PATH="$fakebin:$PATH" "$INSTALLER" "$destination" 2>&1) \ + || fail "installer did not recover from a transient download failure"$'\n'"$out" + [ "$(cat "$tmp/curl-count")" -eq 2 ] || fail "installer did not retry exactly once after recovery" + assert_contains "$out" "download attempt 1 failed; retrying" "installer did not disclose its retry" + [ -x "$destination/shellcheck" ] || fail "installer did not install ShellCheck after retrying" + pass "ShellCheck installer retries a transient download failure" +} + test_rejects_wrong_shellcheck_version() { # Version-independent: a fake shellcheck reporting a different version must be # refused before any lint, proving local and CI cannot silently diverge. @@ -186,6 +242,7 @@ test_ci_invokes_the_owner test_nomistakes_invokes_the_owner test_pins_an_explicit_version test_ci_installs_and_logs_the_pinned_version +test_installer_retries_transient_download_failure test_rejects_wrong_shellcheck_version test_catches_a_real_lint_defect test_ignores_ambient_shellcheck_opts From 4497181ed8a0d2a660cc184610bd2090b9d6f773 Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Wed, 22 Jul 2026 18:34:25 -0700 Subject: [PATCH 06/34] feat(pi): add session-local calm mode (#884) * Add session-local Pi calm mode * no-mistakes(review): Preserve Pi HTML exports during calm mode * no-mistakes(review): Preserve calm exports across submit bindings and share * no-mistakes(document): Document calm-mode feasibility across supported harnesses --- .pi/extensions/fm-calm.ts | 204 +++++++++++++ README.md | 7 +- bin/fm-test-run.sh | 5 +- docs/calm-mode-feasibility.md | 52 ++++ tests/fm-calm-pi-extension.test.sh | 450 +++++++++++++++++++++++++++++ tests/fm-pi-primary-types.test.sh | 12 +- 6 files changed, 723 insertions(+), 7 deletions(-) create mode 100644 .pi/extensions/fm-calm.ts create mode 100644 docs/calm-mode-feasibility.md create mode 100755 tests/fm-calm-pi-extension.test.sh diff --git a/.pi/extensions/fm-calm.ts b/.pi/extensions/fm-calm.ts new file mode 100644 index 0000000..557faf4 --- /dev/null +++ b/.pi/extensions/fm-calm.ts @@ -0,0 +1,204 @@ +// Firstmate's session-local Pi tool-activity presentation toggle. +// +// Compatibility boundary: Pi 0.80.10 exposes built-in ToolDefinitions, per-slot +// renderers, renderShell: "self", session_start replacement reasons, and +// ExtensionUIContext.setToolsExpanded(). The focused tests pin those assumptions. +// Pi renders built-in read images outside those slots and exposes no safe global +// renderer for custom or third-party tools, so those rows intentionally stay visible. +import type { + ExtensionAPI, + ToolDefinition, + ToolRenderResultOptions, +} from "@earendil-works/pi-coding-agent"; +import { + createBashToolDefinition, + createEditToolDefinition, + createFindToolDefinition, + createGrepToolDefinition, + createLsToolDefinition, + createReadToolDefinition, + createWriteToolDefinition, +} from "@earendil-works/pi-coding-agent"; +import { Box, Container, getKeybindings, type Component } from "@earendil-works/pi-tui"; +import type { TSchema } from "typebox"; + +type DefinitionFactory = ( + cwd: string, +) => ToolDefinition; + +type RenderContext = Parameters< + NonNullable["renderCall"]> +>[2]; + +type RenderArgs = Parameters< + NonNullable["renderCall"]> +>[0]; + +type RenderTheme = Parameters< + NonNullable["renderCall"]> +>[1]; + +type RenderResult = Parameters< + NonNullable["renderResult"]> +>[0]; + +type StandardShellState = { + shell?: Box; + call?: Component; + result?: Component; +}; + +export default function (pi: ExtensionAPI) { + let calm = false; + let exportRendering = false; + let removeTerminalInputHandler: (() => void) | undefined; + + function registerBuiltIn( + factory: DefinitionFactory, + ): void { + const definitions = new Map>(); + const definitionFor = (cwd: string): ToolDefinition => { + let definition = definitions.get(cwd); + if (!definition) { + definition = factory(cwd); + definitions.set(cwd, definition); + } + return definition; + }; + + const original = definitionFor(process.cwd()); + const originalRenderCall = original.renderCall; + const originalRenderResult = original.renderResult; + const originalSelfShell = original.renderShell === "self"; + const standardShells = new WeakMap(); + + if (!originalRenderCall || !originalRenderResult) { + throw new Error(`Firstmate calm mode requires both render slots for Pi built-in tool ${original.name}`); + } + + const shellStateFor = ( + context: RenderContext, + ): StandardShellState => { + const rowState = context.state as object; + let shellState = standardShells.get(rowState); + if (!shellState) { + shellState = {}; + standardShells.set(rowState, shellState); + } + return shellState; + }; + + const refreshStandardShell = ( + state: StandardShellState, + theme: RenderTheme, + context: RenderContext, + ): Box => { + const background = context.isPartial + ? (text: string) => theme.bg("toolPendingBg", text) + : context.isError + ? (text: string) => theme.bg("toolErrorBg", text) + : (text: string) => theme.bg("toolSuccessBg", text); + const shell = state.shell ?? new Box(1, 1, background); + state.shell = shell; + shell.setBgFn(background); + shell.clear(); + if (state.call) shell.addChild(state.call); + if (state.result) shell.addChild(state.result); + return shell; + }; + + pi.registerTool({ + ...original, + renderShell: "self", + + async execute(toolCallId, params, signal, onUpdate, ctx) { + return definitionFor(ctx.cwd).execute(toolCallId, params, signal, onUpdate, ctx); + }, + + renderCall( + args: RenderArgs, + theme: RenderTheme, + context: RenderContext, + ) { + if (exportRendering) return originalRenderCall(args, theme, context); + if (calm) return new Container(); + if (originalSelfShell) return originalRenderCall(args, theme, context); + + const state = shellStateFor(context); + state.call = originalRenderCall(args, theme, { + ...context, + lastComponent: state.call, + }); + return refreshStandardShell(state, theme, context); + }, + + renderResult( + result: RenderResult, + options: ToolRenderResultOptions, + theme: RenderTheme, + context: RenderContext, + ) { + if (exportRendering) return originalRenderResult(result, options, theme, context); + if (calm) return new Container(); + if (originalSelfShell) return originalRenderResult(result, options, theme, context); + + const state = shellStateFor(context); + state.result = originalRenderResult(result, options, theme, { + ...context, + lastComponent: state.result, + }); + refreshStandardShell(state, theme, context); + return new Container(); + }, + }); + } + + registerBuiltIn(createReadToolDefinition); + registerBuiltIn(createBashToolDefinition); + registerBuiltIn(createEditToolDefinition); + registerBuiltIn(createWriteToolDefinition); + registerBuiltIn(createGrepToolDefinition); + registerBuiltIn(createFindToolDefinition); + registerBuiltIn(createLsToolDefinition); + + pi.on("session_start", (_event, ctx) => { + calm = false; + exportRendering = false; + removeTerminalInputHandler?.(); + removeTerminalInputHandler = ctx.ui.onTerminalInput((data) => { + if (!calm || !getKeybindings().matches(data, "tui.input.submit")) return; + + const input = ctx.ui.getEditorText().trim(); + if ( + input !== "/share" && + input !== "/export" && + !input.startsWith("/export ") + ) { + return; + } + + exportRendering = true; + setTimeout(() => { + exportRendering = false; + }, 0); + }); + }); + + pi.registerCommand("calm", { + description: + "Toggle built-in call and text-result rows; built-in read images and custom/third-party tool rows stay visible.", + handler: async (_args, ctx) => { + calm = !calm; + + // Setting the current expansion value is Pi's supported transcript-wide + // redraw path. It revisits existing tool rows without changing Ctrl+O state. + ctx.ui.setToolsExpanded(ctx.ui.getToolsExpanded()); + ctx.ui.notify( + calm + ? "Tool activity is hidden where supported; built-in read images and custom/third-party tool rows remain visible." + : "Tool activity is visible.", + "info", + ); + }, + }); +} diff --git a/README.md b/README.md index 52d69cf..54a0b94 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,12 @@ pi ``` For Grok, `--trust` is needed once per clone so project hooks and the turn-end guard load; `/hooks-trust` inside Grok works too. -For Pi, approve the project trust prompt once per clone on first launch so both tracked `.pi/extensions/*.ts` files auto-load. +For Pi, approve the project trust prompt once per clone on first launch so the tracked `.pi/extensions/*.ts` files auto-load. +Every Pi session starts with calm mode off and tool activity visible; `/calm` is a session-local toggle that hides all seven built-in call shells and text-result rows, including existing rows. +Toggling off restores ordinary rendering, and `Ctrl+O` expansion behavior stays unchanged. +Built-in `read` images on image-capable terminals and custom or third-party tool rows remain visible because Pi 0.80.10 does not expose those rows to supported extension renderers. +The toggle changes only interactive rendering, not tool execution, model context, session storage, exports, or diagnostics. +The version-scoped feasibility evidence for keeping this feature Pi-only is recorded in [docs/calm-mode-feasibility.md](docs/calm-mode-feasibility.md). ### Talk to it diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index cac6be7..21233f8 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -117,8 +117,9 @@ now_ms() { # unclassified so new tests are still runnable and visible in summaries. family_for_basename() { case "$1" in - fm-arm-pretool-check.test.sh|fm-brief.test.sh|fm-captain-translation-contract.test.sh|\ - fm-cd-pretool-check.test.sh|fm-composer-ghost.test.sh|fm-composer-lib.test.sh|\ + fm-arm-pretool-check.test.sh|fm-brief.test.sh|fm-calm-pi-extension.test.sh|\ + fm-captain-translation-contract.test.sh|fm-cd-pretool-check.test.sh|\ + fm-composer-ghost.test.sh|fm-composer-lib.test.sh|\ fm-continuity-pretool-check.test.sh|fm-crew-state.test.sh|fm-decision-hold-lifecycle.test.sh|\ fm-dispatch-select.test.sh|fm-ensure-agents-md.test.sh|fm-grok-harness.test.sh|\ fm-herdr-lab.test.sh|fm-instruction-owners.test.sh|fm-lint.test.sh|\ diff --git a/docs/calm-mode-feasibility.md b/docs/calm-mode-feasibility.md new file mode 100644 index 0000000..c10f3b1 --- /dev/null +++ b/docs/calm-mode-feasibility.md @@ -0,0 +1,52 @@ +# Calm-mode harness feasibility + +This document owns the version-scoped feasibility evidence for implementing Firstmate calm mode in each verified harness. +The README owns the user-facing `/calm` usage and limitation contract. + +## Required extension surface + +A qualifying implementation must auto-load from the trusted project, keep the toggle session-local, redraw already-rendered built-in tool rows, restore the harness's ordinary rendering, and leave tool execution, model context, session storage, exports, diagnostics, and expansion state unchanged. +Replacing a built-in tool, patching harness internals, filtering persisted events, or claiming coverage outside a supported renderer does not satisfy that boundary. + +## Verification record + +The following inspection was performed on 2026-07-22. + +```text +$ claude --version +2.1.216 (Claude Code) +$ codex --version +codex-cli 0.144.6 +$ opencode --version +1.17.18 +$ pi --version +0.80.10 +$ grok --version +grok 0.2.106 (bde89716f679) +``` + +The inspected commands were `claude --help`, `claude plugin --help`, `claude plugin validate --help`, `codex --help`, `codex plugin --help`, `codex features list`, `opencode --help`, `opencode debug --help`, `opencode debug config`, `pi --help`, `grok --help`, and `grok plugin --help`. +The inspection also covered the tracked project hook and plugin definitions for all five harnesses and Pi 0.80.10's installed public TypeScript declarations. + +| Harness | Conclusion | Evidence | +| --- | --- | --- | +| Claude Code 2.1.216 | Not feasible through the inspected supported project surface. | Project hooks can observe lifecycle and tool events, while the plugin CLI packages supported components; neither inspected surface exposes a transcript-row renderer or a transcript-wide redraw API. | +| Codex CLI 0.144.6 | Not feasible through the inspected supported project surface. | The tracked hooks expose session, pre-tool, and stop handling, while the plugin and feature inventories expose no TUI tool-row renderer or transcript redraw control. | +| OpenCode 1.17.18 | Not feasible without violating the preservation boundary. | Plugins expose events and tool execution hooks, not a built-in transcript-row renderer. A same-name custom tool can replace a built-in tool, but that changes the tool definition and execution path rather than presentation alone. | +| Pi 0.80.10 | Feasible and implemented. | Public declarations expose `registerTool`, `ToolDefinition.renderCall`, `ToolDefinition.renderResult`, `renderShell`, `setToolsExpanded`, terminal input handling, and extension commands. The focused renderer test and interactive terminal E2E exercise the supported path. | +| Grok CLI 0.2.106 | Not feasible through the inspected supported project surface. | Project hooks expose lifecycle and tool interception, while the plugin CLI exposes no row-renderer contract. `--minimal` changes the session's overall screen mode and does not provide selective, reversible transcript-row control. | + +These conclusions are deliberately limited to the named versions and supported surfaces. +They do not claim that a harness can never add the required renderer API. + +## Pi verification + +`tests/fm-calm-pi-extension.test.sh` compares wrapped and stock Pi renderers, verifies all seven built-ins, exercises already-rendered rows, checks the disclosed image and custom-tool boundaries, covers session reset reasons, proves exports remain ordinary, and drives a genuine interactive terminal session. +`tests/fm-pi-primary-types.test.sh` performs strict no-emit TypeScript checking against the installed Pi 0.80.10 declarations. + +The relevant commands are: + +```sh +tests/fm-calm-pi-extension.test.sh +tests/fm-pi-primary-types.test.sh +``` diff --git a/tests/fm-calm-pi-extension.test.sh b/tests/fm-calm-pi-extension.test.sh new file mode 100755 index 0000000..e1fa030 --- /dev/null +++ b/tests/fm-calm-pi-extension.test.sh @@ -0,0 +1,450 @@ +#!/usr/bin/env bash +# Focused rendering, lifecycle, persistence, and interactive TUI checks for /calm. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TMP_ROOT=$(fm_test_tmproot fm-calm-pi-extension) +EXT="$ROOT/.pi/extensions/fm-calm.ts" +PI_PACKAGE_DIR=${FM_PI_PACKAGE_DIR:-"$(npm root -g 2>/dev/null)/@earendil-works/pi-coding-agent"} +TMUX_SOCKET="fm-calm-$$" +TMUX_SESSION="fm-calm-e2e" + +cleanup() { + if command -v tmux >/dev/null 2>&1; then + tmux -L "$TMUX_SOCKET" kill-server 2>/dev/null || true + fi + fm_test_cleanup +} +trap cleanup EXIT + +wait_for_text() { + local file=$1 text=$2 i=0 + while [ "$i" -lt 120 ]; do + tmux -L "$TMUX_SOCKET" capture-pane -p -t "$TMUX_SESSION" -S - >"$file" 2>/dev/null || true + grep -Fq "$text" "$file" 2>/dev/null && return 0 + sleep 0.05 + i=$((i + 1)) + done + return 1 +} + +test_static_contract() { + local text + assert_present "$EXT" "tracked Pi calm extension is missing" + text=$(cat "$EXT") + assert_contains "$text" 'pi.registerCommand("calm"' "Pi calm extension does not register /calm" + assert_contains "$text" 'pi.on("session_start"' "Pi calm extension does not reset on every session start" + assert_contains "$text" 'calm = false' "Pi calm extension does not default to visible tool activity" + assert_contains "$text" 'ctx.ui.setToolsExpanded(ctx.ui.getToolsExpanded())' "Pi calm extension does not redraw existing rows while preserving Ctrl+O state" + assert_contains "$text" 'ctx.ui.onTerminalInput' "Pi calm extension does not scope hiding to interactive rendering" + assert_contains "$text" 'getKeybindings().matches(data, "tui.input.submit")' "Pi calm export boundary ignores the active submit keybinding" + assert_contains "$text" 'input !== "/share"' "Pi calm export boundary does not cover /share" + assert_contains "$text" 'renderShell: "self"' "Pi calm extension cannot remove the complete tool shell" + assert_contains "$text" 'built-in read images and custom/third-party tool rows stay visible' "Pi calm command description does not disclose both visibility boundaries" + assert_contains "$text" 'built-in read images and custom/third-party tool rows remain visible' "Pi calm enabled status does not disclose both visibility boundaries" + assert_not_contains "$text" 'appendEntry' "Pi calm extension persists its session-local toggle" + assert_not_contains "$text" 'sendMessage' "Pi calm extension changes model context" + for name in Read Bash Edit Write Grep Find Ls; do + assert_contains "$text" "create${name}ToolDefinition" "Pi calm extension does not wrap the $name built-in" + done + pass "Pi calm extension has the default-off, redraw, seven-built-in text-row, and explicit limitation contract" +} + +test_rendering_and_session_lifecycle() { + local fixture out status version + if ! command -v node >/dev/null 2>&1 || ! command -v npm >/dev/null 2>&1; then + echo "skip: node or npm not found for Pi calm renderer test" + return 0 + fi + if [ ! -f "$PI_PACKAGE_DIR/package.json" ]; then + echo "skip: installed @earendil-works/pi-coding-agent package not found" + return 0 + fi + version=$(node -p "require('$PI_PACKAGE_DIR/package.json').version") + [ "$version" = "0.80.10" ] || fail "Pi calm compatibility assumptions require Pi 0.80.10, found $version" + + fixture="$TMP_ROOT/renderer" + mkdir -p "$fixture/node_modules/@earendil-works" + cp "$EXT" "$fixture/fm-calm.ts" + ln -s "$PI_PACKAGE_DIR" "$fixture/node_modules/@earendil-works/pi-coding-agent" + ln -s "$PI_PACKAGE_DIR/node_modules/@earendil-works/pi-tui" "$fixture/node_modules/@earendil-works/pi-tui" + ln -s "$PI_PACKAGE_DIR/node_modules/typebox" "$fixture/node_modules/typebox" + printf '%s\n' '{"type":"module"}' >"$fixture/package.json" + + out=$(cd "$fixture" && EXT="$fixture/fm-calm.ts" PI_PACKAGE_DIR="$PI_PACKAGE_DIR" node --input-type=module 2>&1 <<'JS' +import { writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +const packageRoot = process.env.PI_PACKAGE_DIR; +const [{ ToolExecutionComponent }, { initTheme, theme }, { Text, getKeybindings, setCapabilities }, { createToolHtmlRenderer }] = await Promise.all([ + import(pathToFileURL(`${packageRoot}/dist/modes/interactive/components/tool-execution.js`).href), + import(pathToFileURL(`${packageRoot}/dist/modes/interactive/theme/theme.js`).href), + import(pathToFileURL(`${packageRoot}/node_modules/@earendil-works/pi-tui/dist/index.js`).href), + import(pathToFileURL(`${packageRoot}/dist/core/export-html/tool-renderer.js`).href), +]); +initTheme("dark"); +setCapabilities({ images: null, trueColor: true, hyperlinks: false }); + +const tools = []; +const handlers = new Map(); +let calmCommand; +const pi = { + appendEntry() { + throw new Error("calm mode must not persist extension state"); + }, + on(event, handler) { + handlers.set(event, handler); + }, + registerCommand(name, command) { + if (name === "calm") calmCommand = command; + }, + registerTool(tool) { + tools.push(tool); + }, +}; +const extension = await import(`${pathToFileURL(process.env.EXT).href}?test=${Date.now()}`); +extension.default(pi); + +const names = tools.map((tool) => tool.name); +const expectedNames = ["read", "bash", "edit", "write", "grep", "find", "ls"]; +if (JSON.stringify(names) !== JSON.stringify(expectedNames)) { + throw new Error(`unexpected wrapped built-ins: ${names.join(",")}`); +} +if (!calmCommand || !handlers.has("session_start")) { + throw new Error("calm command or session lifecycle handler was not registered"); +} +if ( + calmCommand.description !== + "Toggle built-in call and text-result rows; built-in read images and custom/third-party tool rows stay visible." +) { + throw new Error(`calm command description does not disclose both visibility boundaries: ${calmCommand.description}`); +} + +writeFileSync("sample.txt", "alpha\n"); +const cases = [ + ["read", { path: "sample.txt" }, { content: [{ type: "text", text: "alpha" }], details: {}, isError: false }], + ["bash", { command: "printf 'CALM_RENDER_OUTPUT\\n'" }, { content: [{ type: "text", text: "CALM_RENDER_OUTPUT" }], details: {}, isError: false }], + ["edit", { path: "sample.txt", edits: [{ oldText: "alpha", newText: "beta" }] }, { content: [{ type: "text", text: "Successfully replaced 1 block(s) in sample.txt." }], details: { diff: "-alpha\n+beta", patch: "", firstChangedLine: 1 }, isError: false }], + ["write", { path: "sample.txt", content: "beta\n" }, { content: [{ type: "text", text: "Successfully wrote 5 bytes to sample.txt" }], details: undefined, isError: false }], + ["grep", { pattern: "alpha", path: "." }, { content: [{ type: "text", text: "sample.txt:1:alpha" }], details: {}, isError: false }], + ["find", { pattern: "*.txt", path: "." }, { content: [{ type: "text", text: "sample.txt" }], details: {}, isError: false }], + ["ls", { path: "." }, { content: [{ type: "text", text: "sample.txt" }], details: {}, isError: false }], +]; +const renderUi = { requestRender() {} }; +const rows = []; +for (const [name, args, result] of cases) { + const wrapped = tools.find((tool) => tool.name === name); + const baseline = new ToolExecutionComponent(name, `baseline-${name}`, args, { showImages: false }, undefined, renderUi, process.cwd()); + const actual = new ToolExecutionComponent(name, `wrapped-${name}`, args, { showImages: false }, wrapped, renderUi, process.cwd()); + for (const row of [baseline, actual]) { + row.markExecutionStarted(); + row.setArgsComplete(); + row.updateResult(result); + } + const collapsedExpected = baseline.render(100); + const collapsedActual = actual.render(100); + if (JSON.stringify(collapsedActual) !== JSON.stringify(collapsedExpected)) { + throw new Error(`${name} collapsed rendering changed while calm mode was off`); + } + baseline.setExpanded(true); + actual.setExpanded(true); + const expandedExpected = baseline.render(100); + const expandedActual = actual.render(100); + if (JSON.stringify(expandedActual) !== JSON.stringify(expandedExpected)) { + throw new Error(`${name} expanded rendering changed while calm mode was off`); + } + rows.push({ name, baseline, actual }); +} + +const customDefinition = { + name: "third_party_tool", + label: "Third party tool", + description: "Custom-tool boundary probe", + parameters: { type: "object", properties: {} }, + renderShell: "self", + async execute() { + return { content: [{ type: "text", text: "CUSTOM_RESULT" }], details: {} }; + }, + renderCall() { + return new Text("CUSTOM_CALL", 0, 0); + }, + renderResult() { + return new Text("CUSTOM_RESULT", 0, 0); + }, +}; +const customRow = new ToolExecutionComponent( + "third_party_tool", + "custom-row", + {}, + { showImages: false }, + customDefinition, + renderUi, + process.cwd(), +); +customRow.markExecutionStarted(); +customRow.setArgsComplete(); +customRow.updateResult({ content: [{ type: "text", text: "CUSTOM_RESULT" }], details: {}, isError: false }); + +setCapabilities({ images: "iterm2", trueColor: true, hyperlinks: true }); +const imageRow = new ToolExecutionComponent( + "read", + "read-image-row", + { path: "pixel.png" }, + { showImages: true }, + tools.find((tool) => tool.name === "read"), + renderUi, + process.cwd(), +); +imageRow.markExecutionStarted(); +imageRow.setArgsComplete(); +imageRow.updateResult({ + content: [ + { + type: "image", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", + mimeType: "image/png", + }, + ], + details: {}, + isError: false, +}); +imageRow.setExpanded(true); +const imageVisibleBefore = imageRow.render(100); +if (!imageVisibleBefore.join("\n").includes("\x1b]1337;File=")) { + throw new Error("image-capable Pi fixture did not render the built-in read image boundary"); +} + +let expanded = true; +let notification = ""; +let editorText = ""; +let terminalInputHandler; +const sessionEntries = [{ type: "message", message: { role: "toolResult", content: "kept" } }]; +const entriesBefore = JSON.stringify(sessionEntries); +const commandContext = { + sessionManager: { getEntries: () => sessionEntries }, + ui: { + getEditorText: () => editorText, + getToolsExpanded: () => expanded, + onTerminalInput(handler) { + terminalInputHandler = handler; + return () => { + if (terminalInputHandler === handler) terminalInputHandler = undefined; + }; + }, + setToolsExpanded(value) { + if (value !== expanded) throw new Error("/calm changed the ordinary Ctrl+O expansion state"); + for (const row of rows) row.actual.setExpanded(value); + customRow.setExpanded(value); + imageRow.setExpanded(value); + }, + notify(message) { + notification = message; + }, + }, +}; + +await handlers.get("session_start")({ reason: "startup" }, commandContext); +await calmCommand.handler("", commandContext); +async function assertStockHtmlRendering(command, submitData) { + editorText = command; + terminalInputHandler(submitData); + const htmlRenderer = createToolHtmlRenderer({ + getToolDefinition: (name) => tools.find((tool) => tool.name === name), + theme, + cwd: process.cwd(), + }); + for (const [name, args, result] of cases.filter(([toolName]) => toolName === "grep" || toolName === "find")) { + const toolCallId = `${command}-${name}`; + const callHtml = htmlRenderer.renderCall(toolCallId, name, args); + const resultHtml = htmlRenderer.renderResult( + toolCallId, + name, + result.content, + result.details, + result.isError, + ); + if (!callHtml || !resultHtml?.expanded) { + throw new Error(`${name} disappeared from ${command} HTML while calm mode was on`); + } + } + editorText = ""; + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +await assertStockHtmlRendering("/export calm.html", "\r"); +getKeybindings().setUserBindings({ "tui.input.submit": "alt+s" }); +editorText = "/export remapped.html"; +terminalInputHandler("\r"); +const unmatchedRenderer = createToolHtmlRenderer({ + getToolDefinition: (name) => tools.find((tool) => tool.name === name), + theme, + cwd: process.cwd(), +}); +if (unmatchedRenderer.renderCall("unmatched-submit", "grep", { pattern: "alpha", path: "." })) { + throw new Error("ordinary non-submit input activated HTML export rendering"); +} +editorText = ""; +await assertStockHtmlRendering("/share", "\x1bs"); +for (const { name, actual } of rows) { + const rendered = actual.render(100); + if (rendered.length !== 0) { + throw new Error(`${name} left residual tool rows while calm mode was on: ${JSON.stringify(rendered)}`); + } +} +const calmImageOutput = imageRow.render(100).join("\n"); +if (!calmImageOutput.includes("\x1b]1337;File=")) { + throw new Error("calm mode hid the disclosed built-in read image boundary"); +} +if (calmImageOutput.includes("pixel.png")) { + throw new Error("calm mode left the built-in read call shell beside the disclosed image output"); +} +if (!customRow.render(100).join("\n").includes("CUSTOM_CALL")) { + throw new Error("calm mode incorrectly claimed or applied custom-tool coverage"); +} +if ( + notification !== + "Tool activity is hidden where supported; built-in read images and custom/third-party tool rows remain visible." +) { + throw new Error(`unexpected hidden status: ${notification}`); +} +if (JSON.stringify(sessionEntries) !== entriesBefore) { + throw new Error("calm mode changed session entries or model context"); +} + +// The image-boundary probe changed terminal capabilities after the original +// stock renders, so refresh the stock comparison under the same capabilities. +for (const { baseline } of rows) baseline.setExpanded(expanded); +await calmCommand.handler("", commandContext); +for (const { name, baseline, actual } of rows) { + if (JSON.stringify(actual.render(100)) !== JSON.stringify(baseline.render(100))) { + throw new Error(`${name} did not restore the expanded standard renderer`); + } +} +if (JSON.stringify(imageRow.render(100)) !== JSON.stringify(imageVisibleBefore)) { + throw new Error("built-in read image row did not restore its ordinary call shell and image output"); +} +if (notification !== "Tool activity is visible.") { + throw new Error(`unexpected visible status: ${notification}`); +} + +for (const reason of ["startup", "new", "resume", "fork", "reload"]) { + await calmCommand.handler("", commandContext); + await handlers.get("session_start")({ reason }, commandContext); + for (const row of rows) row.actual.setExpanded(expanded); + for (const { name, baseline, actual } of rows) { + if (JSON.stringify(actual.render(100)) !== JSON.stringify(baseline.render(100))) { + throw new Error(`${reason} session did not begin with calm mode off for ${name}`); + } + } +} + +const readWrapper = tools.find((tool) => tool.name === "read"); +const { createReadToolDefinition } = await import(pathToFileURL(`${packageRoot}/dist/index.js`).href); +const originalRead = createReadToolDefinition(process.cwd()); +const executeContext = { cwd: process.cwd() }; +const [originalResult, wrappedResult] = await Promise.all([ + originalRead.execute("original-read", { path: "sample.txt" }, undefined, undefined, executeContext), + readWrapper.execute("wrapped-read", { path: "sample.txt" }, undefined, undefined, executeContext), +]); +if (JSON.stringify(wrappedResult) !== JSON.stringify(originalResult)) { + throw new Error("calm wrapper changed built-in read execution or result data"); +} +JS +) + status=$? + [ "$status" -eq 0 ] || fail "Pi calm renderer and lifecycle contract failed: $out" + [ -z "$out" ] || fail "Pi calm renderer test printed output: $out" + pass "Pi calm preserves standard rendering and execution, hides seven built-in call and text rows, keeps read images and custom rows visible, and resets per session" +} + +test_interactive_terminal_e2e() { + local project config session_file export_file default_snapshot expanded_snapshot hidden_snapshot export_snapshot restored_snapshot hash_before hash_after now version + if ! command -v pi >/dev/null 2>&1 || ! command -v tmux >/dev/null 2>&1; then + echo "skip: pi or tmux not found for Pi calm interactive E2E" + return 0 + fi + version=$(pi --version 2>/dev/null || true) + [ "$version" = "0.80.10" ] || fail "Pi calm interactive E2E requires Pi 0.80.10, found $version" + + project="$TMP_ROOT/e2e-project" + config="$TMP_ROOT/e2e-config" + session_file="$TMP_ROOT/calm-session.jsonl" + export_file="$TMP_ROOT/calm-export.html" + default_snapshot="$TMP_ROOT/default.txt" + expanded_snapshot="$TMP_ROOT/expanded.txt" + hidden_snapshot="$TMP_ROOT/hidden.txt" + export_snapshot="$TMP_ROOT/export.txt" + restored_snapshot="$TMP_ROOT/restored.txt" + mkdir -p "$project/.pi/extensions" "$config" + cp "$EXT" "$project/.pi/extensions/fm-calm.ts" + printf '%s\n' '{"tui.input.submit":"alt+s"}' >"$config/keybindings.json" + now=$(date -u +%Y-%m-%dT%H:%M:%S.000Z) + cat >"$session_file" <