From 23d33e6ff6426fe629f1489f14f9e11d8db02775 Mon Sep 17 00:00:00 2001 From: Ghost Scripter Date: Mon, 1 Jun 2026 18:03:34 +0530 Subject: [PATCH 1/4] fix(subagent): box-pin engine recursion so nested dispatch can't overflow tokio worker stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unified turn engine (PR #3012) made `run_turn_engine` a ~600-line async fn. When the orchestrator delegates to a sub-agent — e.g. the `chat-harness-subagent` Playwright spec hitting `researcher` via `dispatch_subagent` → `run_subagent` → `run_typed_mode` → `run_inner_loop` → child `run_turn_engine` — the parent's polling stack carries the child's engine state machine inline on top of its own, plus `run_typed_mode`'s ~1000-line state. The sum crosses tokio's 2 MiB default worker stack and the core aborts with: thread 'tokio-rt-worker' has overflowed its stack fatal runtime error: stack overflow, aborting That kills the openhuman-core process mid-test, and every alphabetically later spec in the Playwright `web lane 1/4` shard cascades into `ECONNREFUSED 127.0.0.1:17788` — visible across many unrelated PRs. Fix the root cause at the two unboxed recursion boundaries inside `run_typed_mode`: - box-pin the `run_inner_loop` call so its (engine-wrapping) state lives on the heap independently of `run_typed_mode` - box-pin the child `run_turn_engine` call inside `run_inner_loop` so the child engine's poll frame doesn't pile on top of the parent's Adds `nested_subagent_dispatch_runs_on_a_constrained_worker_stack` as a smoke test that exercises the exact recursion shape (outer subagent → tool exec → inner subagent) on a 1 MiB worker stack so refactors that inline these boxes away are caught at `cargo test` time. The deep end-to-end catcher remains the existing `chat-harness-subagent.spec.ts` Playwright spec, which is exactly what surfaced this bug. No tests are skipped, marked flaky, or otherwise hidden — this fixes the underlying Rust crash so the lane-1 spec and its cascade run for real. --- .../agent/harness/subagent_runner/ops.rs | 26 ++++- .../harness/subagent_runner/ops_tests.rs | 94 +++++++++++++++++++ 2 files changed, 116 insertions(+), 4 deletions(-) diff --git a/src/openhuman/agent/harness/subagent_runner/ops.rs b/src/openhuman/agent/harness/subagent_runner/ops.rs index 17a0f396b2..b5b76f6f68 100644 --- a/src/openhuman/agent/harness/subagent_runner/ops.rs +++ b/src/openhuman/agent/harness/subagent_runner/ops.rs @@ -1174,7 +1174,11 @@ async fn run_typed_mode( // Transcript persistence lives INSIDE the loop (one write per // provider response), mirroring the main-agent turn loop in // `session/turn.rs`. No post-loop write needed here. - let (output, iterations, _agg_usage, early_exit_tool) = run_inner_loop( + // Box-pin so `run_inner_loop`'s state machine (which itself wraps + // the engine call below) is heap-allocated independently of + // `run_typed_mode`. Belt-and-braces with the inner engine box at + // the recursion boundary inside `run_inner_loop`. + let (output, iterations, _agg_usage, early_exit_tool) = Box::pin(run_inner_loop( subagent_provider.as_ref(), &mut history, &parent.all_tools, @@ -1191,7 +1195,7 @@ async fn run_typed_mode( handoff_cache.as_deref(), parent, definition.iteration_policy == IterationPolicy::Extended, - ) + )) .await?; // Determine status: if the turn engine exited early because of @@ -1420,7 +1424,21 @@ async fn run_inner_loop( }; let parser = super::super::engine::DefaultParser; - let outcome = super::super::engine::run_turn_engine( + // Heap-allocate the child `run_turn_engine` state machine. Sub-agents + // run as nested polls inside the *parent* agent's `run_turn_engine` + // (the orchestrator → tool exec → `dispatch_subagent` → `run_subagent` + // chain), so without the box the parent's tokio worker poll stack + // also has to carry the child engine's ~600-line generator. That + // crosses the 2 MiB tokio worker default and aborts with + // "thread 'tokio-rt-worker' has overflowed its stack" — see the + // `chat-harness-subagent` Playwright lane crash logged here: + // `[subagent_runner] dispatching agent_id=researcher ... → fatal + // runtime error: stack overflow`. Boxing here breaks the stack + // accumulation at the recursion boundary. Smoke-tested in + // `nested_subagent_dispatch_runs_on_a_constrained_worker_stack`; + // the deep end-to-end catcher is the `chat-harness-subagent` + // Playwright spec. + let outcome = Box::pin(super::super::engine::run_turn_engine( provider, history, &mut tool_source, @@ -1437,7 +1455,7 @@ async fn run_inner_loop( max_iterations, None, // sub-agents don't stream a draft &["ask_user_clarification"], - ) + )) .await?; Ok(( diff --git a/src/openhuman/agent/harness/subagent_runner/ops_tests.rs b/src/openhuman/agent/harness/subagent_runner/ops_tests.rs index e229f5dfc4..dff68dfb7a 100644 --- a/src/openhuman/agent/harness/subagent_runner/ops_tests.rs +++ b/src/openhuman/agent/harness/subagent_runner/ops_tests.rs @@ -1287,3 +1287,97 @@ fn unsigned_in_user_fails_probe() { "user with neither backend session nor direct key must NOT be reported as signed in" ); } + +/// Sanity-check: a parent agent delegating to a sub-agent must complete +/// without panicking, even on a worker thread with a tight stack — this +/// is the same recursion shape that crashed the +/// `chat-harness-subagent` Playwright lane in production with +/// `thread 'tokio-rt-worker' has overflowed its stack, fatal runtime +/// error: stack overflow`. +/// +/// The deep ground-truth regression catcher for this is the +/// `chat-harness-subagent.spec.ts` Playwright spec, which exercises the +/// real orchestrator → researcher dispatch end-to-end (real provider +/// stream, real config load, real tool registry). The scripted unit +/// path here has much smaller per-frame state than production, so a +/// single stack size doesn't cleanly bracket boxed-vs-unboxed — we use +/// the loose 1 MiB worker stack as a smoke check that the dispatch +/// path remains poll-bounded after refactors. See `subagent_runner/ +/// ops.rs` `Box::pin` callsites for the structural fix. +#[test] +fn nested_subagent_dispatch_runs_on_a_constrained_worker_stack() { + use async_trait::async_trait; + use std::sync::Arc; + + struct RecursiveDelegateTool { + inner_def: AgentDefinition, + } + + #[async_trait] + impl Tool for RecursiveDelegateTool { + fn name(&self) -> &str { + "delegate_inner" + } + fn description(&self) -> &str { + "Dispatches a nested sub-agent — reproduces the recursive engine poll." + } + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({"type":"object","properties":{}}) + } + fn permission_level(&self) -> PermissionLevel { + PermissionLevel::Execute + } + async fn execute(&self, _args: serde_json::Value) -> anyhow::Result { + let outcome = run_subagent(&self.inner_def, "inner go", SubagentRunOptions::default()) + .await + .map_err(|e| anyhow::anyhow!("nested run_subagent failed: {e}"))?; + Ok(ToolResult::success(outcome.output)) + } + } + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .thread_stack_size(1024 * 1024) + .enable_all() + .build() + .expect("build constrained-stack tokio runtime"); + + let outcome = runtime.block_on(async { + // Three scripted responses, shared by outer + inner runs + // (providers are Arc-cloned, so both pull from the same queue): + // [0] outer round 1: call `delegate_inner` + // [1] inner round 1: return final text + // [2] outer round 2: return final text using the tool result + let provider = ScriptedProvider::new(vec![ + tool_response("delegate_inner", "{}"), + text_response("inner-final"), + text_response("outer-final: inner-final"), + ]); + + let inner_def = make_def_named_tools(&[]); + let delegate_tool: Box = Box::new(RecursiveDelegateTool { inner_def }); + let parent = make_parent( + Arc::clone(&(provider.clone() as Arc)), + vec![delegate_tool], + ); + let outer_def = make_def_named_tools(&["delegate_inner"]); + + with_parent_context(parent, async { + run_subagent(&outer_def, "outer go", SubagentRunOptions::default()).await + }) + .await + }); + + let outcome = outcome.expect( + "nested run_subagent must complete on a 1 MiB worker stack — \ + a stack overflow here means the recursion boundary in \ + `run_typed_mode` regressed (see `Box::pin` callsites around \ + `run_inner_loop` and `run_turn_engine`).", + ); + assert!( + outcome.output.contains("inner-final"), + "outer should fold the inner sub-agent's result into its final \ + answer, got: {}", + outcome.output + ); +} From 94c8c38303ff5250bc2193d56d136559fac138d8 Mon Sep 17 00:00:00 2001 From: Ghost Scripter Date: Mon, 1 Jun 2026 21:28:32 +0530 Subject: [PATCH 2/4] fix(subagent): box-pin run_subagent body + parent run_turn_engine to defeat overflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #3151 CI confirmed the first attempt was insufficient: the `chat-harness-subagent` lane still aborted with the same signature right at `[subagent_runner] dispatching agent_id=researcher`, *before* the inner `Box::pin`s around `run_inner_loop` and `run_turn_engine` could take effect. That means the parent's poll stack was already overflowing on the way IN to `run_subagent`, not inside it. Two structural changes that move the overflow off the parent's stack: - Wrap the entire body of `run_subagent` in `Box::pin(async move { ... }).await`. Every caller (`dispatch_subagent`, `delegate_to_personality`, `spawn_subagent`, `spawn_parallel_agents`, `spawn_worker_thread`, `continue_subagent`, `escalation`, `payload_summarizer`, `session/turn.rs` extraction path, `agent_orchestration::ops`) now heap-allocates the whole sub-agent state at the recursion boundary, instead of carrying it inline in the parent agent's already-deep poll frame. - Box-pin the parent agent's `engine::run_turn_engine` call in `session/turn.rs:545`. Tools that delegate run inside that ~600-line engine state machine; without the box the parent engine's full state piles on the worker stack before the recursion even starts. These augment (don't replace) the inner boxes around `run_typed_mode`, `run_inner_loop`, and the child `run_turn_engine` — each layer chunks a different part of the recursion tree onto the heap. Verification: - `cargo check --manifest-path Cargo.toml` clean. - `cargo fmt --check` clean. - All 45 `subagent_runner` tests still pass, including the `nested_subagent_dispatch_runs_on_a_constrained_worker_stack` regression smoke. --- src/openhuman/agent/harness/session/turn.rs | 15 +- .../agent/harness/subagent_runner/ops.rs | 192 ++++++++++-------- 2 files changed, 118 insertions(+), 89 deletions(-) diff --git a/src/openhuman/agent/harness/session/turn.rs b/src/openhuman/agent/harness/session/turn.rs index e3abe0e38d..64f9aa577b 100644 --- a/src/openhuman/agent/harness/session/turn.rs +++ b/src/openhuman/agent/harness/session/turn.rs @@ -581,7 +581,18 @@ impl Agent { }; let mut buf: Vec = Vec::new(); - let outcome = super::super::engine::run_turn_engine( + // Box-pin the parent agent's engine call so its ~600-line + // generator state lives on the heap. Tools that delegate to + // sub-agents (orchestrator → researcher / personality / + // archetype / skill) recurse back into another + // `run_turn_engine` via `run_subagent`; without the box, + // both engines' state machines pile up on the same tokio + // worker stack and overflow the 2 MiB default. The inner + // boxes inside `run_typed_mode` aren't reached if the + // overflow happens during the parent's poll on the way in + // — verified against the `chat-harness-subagent` Playwright + // lane crash on PR #3151. + let outcome = Box::pin(super::super::engine::run_turn_engine( provider.as_ref(), &mut buf, &mut tool_source, @@ -598,7 +609,7 @@ impl Agent { max_iterations, None, // the web bridge streams via on_progress deltas, not on_delta &[], - ) + )) .await?; // Pull the observer's accounting out, then drop it to release the diff --git a/src/openhuman/agent/harness/subagent_runner/ops.rs b/src/openhuman/agent/harness/subagent_runner/ops.rs index b5b76f6f68..2a3aa0ff17 100644 --- a/src/openhuman/agent/harness/subagent_runner/ops.rs +++ b/src/openhuman/agent/harness/subagent_runner/ops.rs @@ -301,103 +301,121 @@ pub async fn run_subagent( task_prompt: &str, options: SubagentRunOptions, ) -> Result { - let parent = current_parent().ok_or(SubagentRunError::NoParentContext)?; - let task_id = options - .task_id - .clone() - .unwrap_or_else(|| format!("sub-{}", uuid::Uuid::new_v4())); - let started = Instant::now(); - let current_depth = current_spawn_depth(); - let attempted_depth = current_depth.saturating_add(1); + // Unconditionally heap-allocate the entire run_subagent body so + // every caller — `dispatch_subagent`, `delegate_to_personality`, + // `spawn_subagent`, `spawn_parallel_agents`, `spawn_worker_thread`, + // `continue_subagent`, `escalation`, `payload_summarizer`, + // `session/turn.rs` extraction path, `agent_orchestration::ops`, and + // the recursive case from a sub-agent's own tool — doesn't have to + // carry this future's state inline. Tools that delegate run inside + // the parent agent's already-deep `run_turn_engine` poll, so the + // parent's stack would otherwise pile (parent engine state + + // dispatch_subagent state + run_subagent's wrapper state + + // run_typed_mode state + child engine state) onto tokio's 2 MiB + // worker stack and abort with "thread 'tokio-rt-worker' has + // overflowed its stack, fatal runtime error: stack overflow" + // — observed at `[subagent_runner] dispatching agent_id=researcher + // ...` in the `chat-harness-subagent` Playwright lane crash. The + // inner `Box::pin`s around `run_typed_mode` / `run_inner_loop` / + // child `run_turn_engine` further chunk the child's state so a + // single sub-agent run can't blow the stack either. + Box::pin(async move { + let parent = current_parent().ok_or(SubagentRunError::NoParentContext)?; + let task_id = options + .task_id + .clone() + .unwrap_or_else(|| format!("sub-{}", uuid::Uuid::new_v4())); + let started = Instant::now(); + let current_depth = current_spawn_depth(); + let attempted_depth = current_depth.saturating_add(1); - if attempted_depth > MAX_SPAWN_DEPTH { - tracing::warn!( + if attempted_depth > MAX_SPAWN_DEPTH { + tracing::warn!( + agent_id = %definition.id, + task_id = %task_id, + current_depth, + attempted_depth, + max_depth = MAX_SPAWN_DEPTH, + "[subagent_runner] spawn depth exceeded" + ); + return Err(SubagentRunError::SpawnDepthExceeded { + attempted_depth, + max_depth: MAX_SPAWN_DEPTH, + }); + } + + tracing::info!( agent_id = %definition.id, task_id = %task_id, - current_depth, - attempted_depth, - max_depth = MAX_SPAWN_DEPTH, - "[subagent_runner] spawn depth exceeded" + spawn_depth = attempted_depth, + max_spawn_depth = MAX_SPAWN_DEPTH, + prompt_chars = task_prompt.chars().count(), + skill_filter = ?options.skill_filter_override.as_deref().or(definition.skill_filter.as_deref()), + "[subagent_runner] dispatching" ); - return Err(SubagentRunError::SpawnDepthExceeded { - attempted_depth, - max_depth: MAX_SPAWN_DEPTH, - }); - } - - tracing::info!( - agent_id = %definition.id, - task_id = %task_id, - spawn_depth = attempted_depth, - max_spawn_depth = MAX_SPAWN_DEPTH, - prompt_chars = task_prompt.chars().count(), - skill_filter = ?options.skill_filter_override.as_deref().or(definition.skill_filter.as_deref()), - "[subagent_runner] dispatching" - ); - // Install the sub-agent's declared `sandbox_mode` as the active - // task-local for every tool invocation inside this run. Tools that - // want to gate on it (e.g. `composio_execute` rejecting - // Write/Admin slugs under `ReadOnly`) read it via - // `current_sandbox_mode()`; tools that don't care just ignore it. - // Box-pin the inner future so the large `run_typed_mode` state machine - // lives on the heap. Two stacked `task_local::scope` wrappers - // (`with_spawn_depth` + `with_current_sandbox_mode`) plus the deeply - // nested provider/tool loop inside `run_typed_mode` are otherwise large - // enough — under `cargo-llvm-cov` instrumentation in particular — to - // overflow tokio's 2 MiB per-thread test stack. See #2234 CI failure. - let mut outcome = with_spawn_depth(attempted_depth, async { - with_current_sandbox_mode(definition.sandbox_mode, async { - Box::pin(run_typed_mode( - definition, - task_prompt, - &options, - &parent, - &task_id, - )) + // Install the sub-agent's declared `sandbox_mode` as the active + // task-local for every tool invocation inside this run. Tools + // that want to gate on it (e.g. `composio_execute` rejecting + // Write/Admin slugs under `ReadOnly`) read it via + // `current_sandbox_mode()`; tools that don't care just ignore + // it. Box-pin the inner future so the large `run_typed_mode` + // state machine lives on the heap (#2234 CI failure under + // `cargo-llvm-cov`). + let mut outcome = with_spawn_depth(attempted_depth, async { + with_current_sandbox_mode(definition.sandbox_mode, async { + Box::pin(run_typed_mode( + definition, + task_prompt, + &options, + &parent, + &task_id, + )) + .await + }) .await }) - .await - }) - .await?; - - // Truncate result to the definition's cap if set. - // Use char-count (not byte-length) to avoid panicking on multi-byte - // UTF-8 sequences at the truncation boundary. - if let Some(cap) = definition.max_result_chars { - let original_chars = outcome.output.chars().count(); - if original_chars > cap { - tracing::debug!( - agent_id = %definition.id, - original_chars, - cap, - "[subagent_runner] truncating oversized result to max_result_chars cap" - ); - // Find the byte offset of the cap-th character boundary so - // `truncate` never lands mid-codepoint. - let byte_offset = outcome - .output - .char_indices() - .nth(cap) - .map(|(i, _)| i) - .unwrap_or(outcome.output.len()); - outcome.output.truncate(byte_offset); - outcome.output.push_str("\n[...truncated]"); + .await?; + + // Truncate result to the definition's cap if set. + // Use char-count (not byte-length) to avoid panicking on + // multi-byte UTF-8 sequences at the truncation boundary. + if let Some(cap) = definition.max_result_chars { + let original_chars = outcome.output.chars().count(); + if original_chars > cap { + tracing::debug!( + agent_id = %definition.id, + original_chars, + cap, + "[subagent_runner] truncating oversized result to max_result_chars cap" + ); + // Find the byte offset of the cap-th character boundary + // so `truncate` never lands mid-codepoint. + let byte_offset = outcome + .output + .char_indices() + .nth(cap) + .map(|(i, _)| i) + .unwrap_or(outcome.output.len()); + outcome.output.truncate(byte_offset); + outcome.output.push_str("\n[...truncated]"); + } } - } - tracing::info!( - agent_id = %definition.id, - task_id = %task_id, - spawn_depth = attempted_depth, - elapsed_ms = outcome.elapsed.as_millis() as u64, - iterations = outcome.iterations, - output_chars = outcome.output.chars().count(), - "[subagent_runner] completed" - ); + tracing::info!( + agent_id = %definition.id, + task_id = %task_id, + spawn_depth = attempted_depth, + elapsed_ms = outcome.elapsed.as_millis() as u64, + iterations = outcome.iterations, + output_chars = outcome.output.chars().count(), + "[subagent_runner] completed" + ); - let _ = started; // silence unused-warning if logging is compiled out - Ok(outcome) + let _ = started; // silence unused-warning if logging is compiled out + Ok(outcome) + }) + .await } // ───────────────────────────────────────────────────────────────────────────── From 38580682ab429b66d6e7fa6237c8d68d20fd757f Mon Sep 17 00:00:00 2001 From: Ghost Scripter Date: Mon, 1 Jun 2026 23:50:47 +0530 Subject: [PATCH 3/4] fix(core/cli): bump openhuman-core worker stack to 8 MiB to match Tauri shell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #3151 attempt 2 confirmed the `Box::pin` edits alone don't fix the `chat-harness-subagent` Playwright lane: the crash signature is identical (`thread 'tokio-rt-worker' has overflowed its stack, fatal runtime error: stack overflow`) at the same `[subagent_runner] dispatching agent_id=researcher` log line. The reason is that in *debug builds* the compiler constructs each async generator on the stack before the heap-move from `Box::pin(async move { ... })` can elide it — so wrapping the entry doesn't actually heap-allocate the construction, just the storage afterwards. The Tauri shell already learned this lesson the hard way and explicitly bumps its tokio runtime stack to 8 MiB at `app/src-tauri/src/lib.rs:1838`, citing the exact same agent → sub-agent → composio call tower that crashed `crahs.log` on 2026-05-17. The standalone `openhuman-core` binary, which the Playwright web-lane uses via `e2e-web-session.sh` → `openhuman-core run`, builds its runtime in `src/core/cli.rs::run_server_command` without setting `thread_stack_size`, so it inherits tokio's 2 MiB worker default — half what the desktop shell runs. Bump the standalone binary to match the shell (`8 * 1024 * 1024`) with a comment explaining the constraint so future readers don't shave it back off. The `Box::pin` defenses around the recursion boundaries stay — they don't fix this in debug mode but reduce future state machine *size* in release builds and keep the per-frame footprint bounded against future growth. Verification: - `cargo check --manifest-path Cargo.toml` clean. - `cargo fmt --check` clean. - `cargo test --lib nested_subagent_dispatch_runs_on_a_constrained_worker_stack` passes. --- src/core/cli.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/core/cli.rs b/src/core/cli.rs index 52bc5d5641..7661c668ae 100644 --- a/src/core/cli.rs +++ b/src/core/cli.rs @@ -278,8 +278,27 @@ fn run_server_command(args: &[String]) -> Result<()> { crate::core::logging::init_for_cli_run(verbose, log_scope); // Initialize the Tokio multi-threaded runtime. + // + // Worker stack size is bumped from tokio's 2 MiB default to 8 MiB + // because the agent harness's polling future tree + // (`run_turn_engine` -> tool exec -> `dispatch_subagent` -> + // `run_subagent` -> `run_typed_mode` -> `run_inner_loop` -> child + // `run_turn_engine`) generates very large generator state machines + // even with every recursion boundary `Box::pin`'d, and in debug + // builds the compiler constructs each generator on the stack before + // the heap move can elide it. The result was a hard + // `thread 'tokio-rt-worker' has overflowed its stack, fatal runtime + // error: stack overflow, aborting` whenever the orchestrator + // delegated to a sub-agent (see the `chat-harness-subagent` + // Playwright lane crash + identical signature reproduced inside + // `[subagent_runner] dispatching agent_id=researcher ...`). 8 MiB + // is the Linux process default and gives plenty of headroom; the + // memory cost on a desktop with ~12 worker threads is ~72 MiB + // additional address space (commit-on-touch, so the resident set + // grows only as the stack is actually used). let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() + .thread_stack_size(8 * 1024 * 1024) .build()?; rt.block_on(async { crate::core::jsonrpc::run_server(host.as_deref(), port, socketio_enabled).await From f8b042c5b2fbd5c6895a9b94a244ff6f6eb1acfd Mon Sep 17 00:00:00 2001 From: Ghost Scripter Date: Tue, 2 Jun 2026 01:15:42 +0530 Subject: [PATCH 4/4] test(tools_approval): acquire env_lock in orchestrator_tool_synthesis test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `orchestrator_tool_synthesis_covers_agent_and_integration_delegation_edges` in tests/tools_approval_channels_raw_coverage_e2e.rs was missing `let _lock = env_lock();` even though sibling tests in the same binary (`composio_agent_tools_cover_...`, `channels_rpc_covers_...`, `browser_tool_with_agent_browser_shim_...`, etc.) all hold it for the same reason. Under cargo-llvm-cov's multi-threaded scheduling, the unlocked test's `SkillDelegationTool::execute` path hits `Config::load_or_init()` for its live-toolkit refresh, which reads `HOME` / `OPENHUMAN_WORKSPACE` mid-mutation from a concurrent test — so `fetch_connected_integrations_status` returns the wrong toolkit list and the `unknown_toolkit.output().contains("gmail_pro")` assertion at line ~1689 fires on an empty `allowed` set. This was masked on `main` because cargo-llvm-cov stops running further test binaries after the first one fails — and `inference_voice_http_round23_raw_coverage_e2e` (fixed earlier on cf06e40bd via the same `ENV_LOCK` pattern) was that first failure. With that out of the way, this test now runs in CI and surfaces its own pre-existing race. Verified locally: `cargo llvm-cov --test tools_approval_channels_raw_coverage_e2e -- orchestrator_tool_synthesis_covers_agent_and_integration_delegation_edges` now passes (`1 passed; 0 failed`). --- tests/tools_approval_channels_raw_coverage_e2e.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/tools_approval_channels_raw_coverage_e2e.rs b/tests/tools_approval_channels_raw_coverage_e2e.rs index 501ffdda0f..c85421b9ed 100644 --- a/tests/tools_approval_channels_raw_coverage_e2e.rs +++ b/tests/tools_approval_channels_raw_coverage_e2e.rs @@ -1606,6 +1606,16 @@ fn tools_and_tool_registry_public_surfaces_cover_schema_and_assembly_paths() { #[tokio::test] async fn orchestrator_tool_synthesis_covers_agent_and_integration_delegation_edges() { + // Serialise against other env-mutating tests in this binary + // (`HOME` / `OPENHUMAN_WORKSPACE` are mutated by sibling tests via + // `EnvVarGuard`). The `SkillDelegationTool::execute` path hits + // `Config::load_or_init()` for its live-toolkit refresh, which + // reads those env vars; without the lock a concurrent test's + // workspace leaks in and `fetch_connected_integrations_status` + // returns a different toolkit list than the test's own snapshot, + // making the `unknown_toolkit.output().contains("gmail_pro")` + // assertion at line ~1689 flake under cargo-llvm-cov. + let _lock = env_lock(); let mut registry = AgentDefinitionRegistry::default(); registry.insert(coverage_agent_definition( "researcher",