From 2d86db46f335d50c867f9a197f36ced70f3f3eb1 Mon Sep 17 00:00:00 2001 From: Elisa Martinez Abad Date: Wed, 12 Aug 2026 07:40:42 -0700 Subject: [PATCH 1/6] feat: add global timeout to terminal bench Signed-off-by: Elisa Martinez Abad --- responses_api_agents/anyterminal_agent/app.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/responses_api_agents/anyterminal_agent/app.py b/responses_api_agents/anyterminal_agent/app.py index 3a45392012..7dfa15d432 100644 --- a/responses_api_agents/anyterminal_agent/app.py +++ b/responses_api_agents/anyterminal_agent/app.py @@ -298,6 +298,8 @@ class AnyTerminalAgentConfig(BaseResponsesAPIAgentConfig): sandbox_model_base_url: Optional[str] = None agent_runtime_source: str = "auto" tb_agent_timeout: int = 1800 + # When set, overrides the per-task agent_timeout_sec from the dataset for every task. + global_agent_timeout: Optional[int] = None tb_eval_timeout: int = 300 tb_sandbox_ttl: int = 7200 agent_overhead_mb: int = 2048 # extra container memory on top of the task's memory_mb for the @@ -770,9 +772,11 @@ def _setup_params( agent_run_id = f"{task_name}_{int(time.time())}_{uuid.uuid4().hex[:8]}" - # Per-task timeouts override config defaults when available. + # Per-task timeouts override config defaults when available, unless global_agent_timeout is set. config_overrides = {} - if problem_info.get("agent_timeout_sec"): + if self.config.global_agent_timeout is not None: + config_overrides["tb_agent_timeout"] = self.config.global_agent_timeout + elif problem_info.get("agent_timeout_sec"): config_overrides["tb_agent_timeout"] = int(float(problem_info["agent_timeout_sec"])) if problem_info.get("verifier_timeout_sec"): config_overrides["tb_eval_timeout"] = int(float(problem_info["verifier_timeout_sec"])) From 0e63e6652cb6137126df61831d789bee6a0d8c05 Mon Sep 17 00:00:00 2001 From: Elisa Martinez Abad Date: Thu, 27 Aug 2026 17:48:34 +0200 Subject: [PATCH 2/6] tests(anyterminal): add global timeout tests Signed-off-by: Elisa Martinez Abad --- .../anyterminal_agent/tests/test_app.py | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/responses_api_agents/anyterminal_agent/tests/test_app.py b/responses_api_agents/anyterminal_agent/tests/test_app.py index 4d28ee1edc..c8b2f6a80e 100644 --- a/responses_api_agents/anyterminal_agent/tests/test_app.py +++ b/responses_api_agents/anyterminal_agent/tests/test_app.py @@ -23,7 +23,7 @@ import json from pathlib import Path from types import SimpleNamespace -from unittest.mock import AsyncMock, PropertyMock, patch +from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch import pytest @@ -32,11 +32,13 @@ from nemo_gym.sandbox.providers.apptainer import ApptainerProvider from nemo_gym.sandbox.providers.apptainer import provider as apptainer_provider from nemo_gym.sandbox.providers.docker import DockerProvider +from nemo_gym.server_utils import ServerClient from responses_api_agents.anyterminal_agent.app import ( _RUNNER_TEMPLATE, AnyTerminalAgent, AnyTerminalAgentConfig, AnyTerminalInstanceConfig, + AnyTerminalServerConfig, GymAgentHarnessProcessor, RunTerminalAgent, _build_provider, @@ -161,8 +163,10 @@ def test_example_jsonl_parses(self) -> None: # ── helpers ─────────────────────────────────────────────────────────────────────── -def _make_body(content: str = "solve this") -> NeMoGymResponseCreateParamsNonStreaming: - return NeMoGymResponseCreateParamsNonStreaming(input=[{"role": "user", "content": content}], model="test-model") +def _make_body(content: str = "solve this", **kwargs) -> NeMoGymResponseCreateParamsNonStreaming: + return NeMoGymResponseCreateParamsNonStreaming( + input=[{"role": "user", "content": content}], model="test-model", **kwargs + ) def _make_instance_config(tmp_path: Path, **overrides) -> AnyTerminalInstanceConfig: @@ -230,6 +234,50 @@ def test_toml_without_sections_returns_nones(self, tmp_path: Path) -> None: assert result.get("verifier_timeout_sec") is None +# ── AnyTerminalAgent._setup_params ────────────────────────────────────────────────── + + +def _make_setup_agent(tmp_path: Path, **config_overrides) -> AnyTerminalAgent: + # model_post_init has heavy side effects (deps install, provider resolution) that + # _setup_params doesn't touch, so bypass it and set only what _setup_params reads. + with patch.object(AnyTerminalAgent, "model_post_init", lambda self, context: None): + agent = AnyTerminalAgent(config=_config(**config_overrides), server_client=MagicMock(spec=ServerClient)) + agent._server = AnyTerminalServerConfig( + run_session_id="test_session", + base_results_dir=tmp_path / "results", + model_server_url="", + nemo_gym_root=PARENT_DIR, + agent_deps_dir=tmp_path, + ) + return agent + + +class TestSetupParams: + def test_uses_per_task_timeout_by_default(self, tmp_path: Path) -> None: + agent = _make_setup_agent(tmp_path) + body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path), "agent_timeout_sec": "900"}) + + params = agent._setup_params(body) + + assert params.tb_agent_timeout == 900 + + def test_global_agent_timeout_overrides_per_task_timeout(self, tmp_path: Path) -> None: + agent = _make_setup_agent(tmp_path, global_agent_timeout=7200) + body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path), "agent_timeout_sec": "900"}) + + params = agent._setup_params(body) + + assert params.tb_agent_timeout == 7200 + + def test_global_agent_timeout_applies_without_per_task_timeout(self, tmp_path: Path) -> None: + agent = _make_setup_agent(tmp_path, global_agent_timeout=7200) + body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path)}) + + params = agent._setup_params(body) + + assert params.tb_agent_timeout == 7200 + + # ── _instruction_from_input ─────────────────────────────────────────────────────── From 21aca58c3ddefe03f773216dee640c5e5ada012b Mon Sep 17 00:00:00 2001 From: Riccardo Roveri Date: Thu, 3 Sep 2026 21:56:55 +0200 Subject: [PATCH 3/6] fix(anyterminal): reject global_agent_timeout=0 global_agent_timeout=0 passed validation and was treated as a real timeout (the check was `is not None`), failing every task instantly with an empty trace. Signed-off-by: Riccardo Roveri --- responses_api_agents/anyterminal_agent/app.py | 2 +- responses_api_agents/anyterminal_agent/tests/test_app.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/responses_api_agents/anyterminal_agent/app.py b/responses_api_agents/anyterminal_agent/app.py index 7dfa15d432..43c96e96b2 100644 --- a/responses_api_agents/anyterminal_agent/app.py +++ b/responses_api_agents/anyterminal_agent/app.py @@ -299,7 +299,7 @@ class AnyTerminalAgentConfig(BaseResponsesAPIAgentConfig): agent_runtime_source: str = "auto" tb_agent_timeout: int = 1800 # When set, overrides the per-task agent_timeout_sec from the dataset for every task. - global_agent_timeout: Optional[int] = None + global_agent_timeout: Optional[int] = Field(default=None, gt=0) tb_eval_timeout: int = 300 tb_sandbox_ttl: int = 7200 agent_overhead_mb: int = 2048 # extra container memory on top of the task's memory_mb for the diff --git a/responses_api_agents/anyterminal_agent/tests/test_app.py b/responses_api_agents/anyterminal_agent/tests/test_app.py index c8b2f6a80e..e50dae1152 100644 --- a/responses_api_agents/anyterminal_agent/tests/test_app.py +++ b/responses_api_agents/anyterminal_agent/tests/test_app.py @@ -26,6 +26,7 @@ from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch import pytest +from pydantic import ValidationError from nemo_gym import PARENT_DIR from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming @@ -277,6 +278,10 @@ def test_global_agent_timeout_applies_without_per_task_timeout(self, tmp_path: P assert params.tb_agent_timeout == 7200 + def test_global_agent_timeout_rejects_zero(self) -> None: + with pytest.raises(ValidationError, match="global_agent_timeout"): + _config(global_agent_timeout=0) + # ── _instruction_from_input ─────────────────────────────────────────────────────── From e014437e9998276ea771c53d58dd47809e6e852b Mon Sep 17 00:00:00 2001 From: Riccardo Roveri Date: Thu, 3 Sep 2026 21:58:10 +0200 Subject: [PATCH 4/6] fix(anyterminal): mask samples killed by the sandbox runtime, not just timeouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit exec() on a container the runtime already removed (e.g. TTL-killed) returns error_type="sandbox" without raising, so agent_timed_out/container_timed_out stayed False and mask_sample never fired — the task scored as a real failure instead of being excluded. Matches anyswe_agent's existing error_type in ("timeout", "sandbox") check. Signed-off-by: Riccardo Roveri --- responses_api_agents/anyterminal_agent/app.py | 4 ++-- .../anyterminal_agent/tests/test_app.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/responses_api_agents/anyterminal_agent/app.py b/responses_api_agents/anyterminal_agent/app.py index 43c96e96b2..db23af9a8d 100644 --- a/responses_api_agents/anyterminal_agent/app.py +++ b/responses_api_agents/anyterminal_agent/app.py @@ -545,7 +545,7 @@ async def _run_agent(self, sandbox: AsyncSandbox, cfg: AnyTerminalInstanceConfig if result.return_code != 0: detail = result.stderr or result.stdout or "" print(f"[{cfg.task_name}] agent exit {result.return_code}: {detail[-2000:]}", flush=True) - return time.time() - t0, result.error_type == "timeout" + return time.time() - t0, result.error_type in ("timeout", "sandbox") async def _stage_tests(self, cfg: AnyTerminalInstanceConfig) -> None: """Copy the task's test files into the staging dir, visible to the sandbox at /tests.""" @@ -567,7 +567,7 @@ async def _run_eval(self, sandbox: AsyncSandbox, cfg: AnyTerminalInstanceConfig) result = await sandbox.exec(_apt_root_sandbox(cfg) + test_cmd, timeout_s=cfg.tb_eval_timeout, user="root") if result.return_code != 0: print(f"[{cfg.task_name}] eval exit {result.return_code}: {(result.stderr or '')[-2000:]}", flush=True) - return time.time() - t0, result.error_type == "timeout" + return time.time() - t0, result.error_type in ("timeout", "sandbox") async def process_single_datapoint(self) -> bool: cfg = self.config diff --git a/responses_api_agents/anyterminal_agent/tests/test_app.py b/responses_api_agents/anyterminal_agent/tests/test_app.py index e50dae1152..0b11397d1d 100644 --- a/responses_api_agents/anyterminal_agent/tests/test_app.py +++ b/responses_api_agents/anyterminal_agent/tests/test_app.py @@ -664,6 +664,25 @@ async def test_agent_timeout_sets_flag_and_masks(self, tmp_path: Path) -> None: assert metrics["agent_timed_out"] is True assert metrics["mask_sample"] is True + async def test_container_killed_mid_run_sets_flag_and_masks(self, tmp_path: Path) -> None: + # exec() on a container the TTL already removed returns error_type="sandbox" without + # raising, so this must be caught the same way a "timeout" is, not just via the + # except-block sandbox_failed path (which never fires here). + cfg = _make_instance_config(tmp_path) + sandbox = SimpleNamespace( + start=AsyncMock(), + exec=AsyncMock(return_value=_sandbox_result(return_code=125, error_type="sandbox")), + stop=AsyncMock(), + ) + with patch("responses_api_agents.anyterminal_agent.app.AsyncSandbox", return_value=sandbox): + with patch.object(RunTerminalAgent, "_stage_tests", new=AsyncMock(return_value=None)): + await RunTerminalAgent(config=cfg).process_single_datapoint() + + metrics = json.loads(cfg.metrics_fpath.read_text()) + assert metrics["agent_timed_out"] is True + assert metrics["sandbox_failed"] is False + assert metrics["mask_sample"] is True + async def test_sandbox_start_failure_is_isolated(self, tmp_path: Path) -> None: cfg = _make_instance_config(tmp_path) sandbox = SimpleNamespace( From e5b905fa7a1a16de02ab32d6d98e0ecb092741c3 Mon Sep 17 00:00:00 2001 From: Riccardo Roveri Date: Thu, 3 Sep 2026 21:59:39 +0200 Subject: [PATCH 5/6] fix(anyterminal): reject a timeout config that would outlive the sandbox TTL global_agent_timeout could be set higher than tb_sandbox_ttl with nothing checking it, so the container gets killed mid-run regardless of how much of its own timeout the agent still had left. Fail fast at setup instead. Signed-off-by: Riccardo Roveri --- responses_api_agents/anyterminal_agent/app.py | 9 +++++++++ .../anyterminal_agent/tests/test_app.py | 11 +++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/responses_api_agents/anyterminal_agent/app.py b/responses_api_agents/anyterminal_agent/app.py index db23af9a8d..1b854a95b1 100644 --- a/responses_api_agents/anyterminal_agent/app.py +++ b/responses_api_agents/anyterminal_agent/app.py @@ -781,6 +781,15 @@ def _setup_params( if problem_info.get("verifier_timeout_sec"): config_overrides["tb_eval_timeout"] = int(float(problem_info["verifier_timeout_sec"])) + effective_agent_timeout = config_overrides.get("tb_agent_timeout", self.config.tb_agent_timeout) + effective_eval_timeout = config_overrides.get("tb_eval_timeout", self.config.tb_eval_timeout) + if effective_agent_timeout + effective_eval_timeout >= self.config.tb_sandbox_ttl: + raise ValueError( + f"[{task_name}] tb_agent_timeout ({effective_agent_timeout}) + tb_eval_timeout " + f"({effective_eval_timeout}) >= tb_sandbox_ttl ({self.config.tb_sandbox_ttl}); " + "the container would be killed before the run can finish. Raise tb_sandbox_ttl." + ) + server_config = self._server.model_dump() if not self.config.sandbox_model_base_url and rollout_id and server_config["model_server_url"]: server_config["model_server_url"] = apply_rollout_prefix(server_config["model_server_url"], rollout_id) diff --git a/responses_api_agents/anyterminal_agent/tests/test_app.py b/responses_api_agents/anyterminal_agent/tests/test_app.py index 0b11397d1d..42c019e9ab 100644 --- a/responses_api_agents/anyterminal_agent/tests/test_app.py +++ b/responses_api_agents/anyterminal_agent/tests/test_app.py @@ -263,7 +263,7 @@ def test_uses_per_task_timeout_by_default(self, tmp_path: Path) -> None: assert params.tb_agent_timeout == 900 def test_global_agent_timeout_overrides_per_task_timeout(self, tmp_path: Path) -> None: - agent = _make_setup_agent(tmp_path, global_agent_timeout=7200) + agent = _make_setup_agent(tmp_path, global_agent_timeout=7200, tb_sandbox_ttl=12000) body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path), "agent_timeout_sec": "900"}) params = agent._setup_params(body) @@ -271,13 +271,20 @@ def test_global_agent_timeout_overrides_per_task_timeout(self, tmp_path: Path) - assert params.tb_agent_timeout == 7200 def test_global_agent_timeout_applies_without_per_task_timeout(self, tmp_path: Path) -> None: - agent = _make_setup_agent(tmp_path, global_agent_timeout=7200) + agent = _make_setup_agent(tmp_path, global_agent_timeout=7200, tb_sandbox_ttl=12000) body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path)}) params = agent._setup_params(body) assert params.tb_agent_timeout == 7200 + def test_global_agent_timeout_exceeding_sandbox_ttl_is_rejected(self, tmp_path: Path) -> None: + agent = _make_setup_agent(tmp_path, global_agent_timeout=7200) + body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path)}) + + with pytest.raises(ValueError, match="tb_sandbox_ttl"): + agent._setup_params(body) + def test_global_agent_timeout_rejects_zero(self) -> None: with pytest.raises(ValidationError, match="global_agent_timeout"): _config(global_agent_timeout=0) From 7093900fe7746ac95692aef81fffbf22db7d3be1 Mon Sep 17 00:00:00 2001 From: Riccardo Roveri Date: Fri, 4 Sep 2026 16:04:49 +0200 Subject: [PATCH 6/6] fix(anyterminal): derive tb_sandbox_ttl from the task's own timeouts The fail-fast check correctly caught a timeout config that would outlive the sandbox, but only refused to run rather than fixing it. Derive the container's lifetime from the task's own timeouts instead, mirroring anyswe_agent's existing pattern (swebench_agent_timeout + swebench_tests_timeout + 600) -- every task gets a sandbox that's guaranteed long enough for its own configured timeouts, no manual tb_sandbox_ttl override needed. Signed-off-by: Riccardo Roveri --- responses_api_agents/anyterminal_agent/app.py | 12 +++++----- .../anyterminal_agent/tests/test_app.py | 23 +++++++++++++------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/responses_api_agents/anyterminal_agent/app.py b/responses_api_agents/anyterminal_agent/app.py index 8dce8437d0..d0f8eb7c72 100644 --- a/responses_api_agents/anyterminal_agent/app.py +++ b/responses_api_agents/anyterminal_agent/app.py @@ -835,14 +835,14 @@ def _setup_params( if problem_info.get("verifier_timeout_sec"): config_overrides["tb_eval_timeout"] = int(float(problem_info["verifier_timeout_sec"])) + # The container must outlive the task, or it gets torn down mid-run and the task scores as + # a real failure instead of the infra issue it is. Mirrors anyswe_agent's derivation + # (swebench_agent_timeout + swebench_tests_timeout + 600). effective_agent_timeout = config_overrides.get("tb_agent_timeout", self.config.tb_agent_timeout) effective_eval_timeout = config_overrides.get("tb_eval_timeout", self.config.tb_eval_timeout) - if effective_agent_timeout + effective_eval_timeout >= self.config.tb_sandbox_ttl: - raise ValueError( - f"[{task_name}] tb_agent_timeout ({effective_agent_timeout}) + tb_eval_timeout " - f"({effective_eval_timeout}) >= tb_sandbox_ttl ({self.config.tb_sandbox_ttl}); " - "the container would be killed before the run can finish. Raise tb_sandbox_ttl." - ) + required_ttl = effective_agent_timeout + effective_eval_timeout + 600 + if required_ttl > self.config.tb_sandbox_ttl: + config_overrides["tb_sandbox_ttl"] = required_ttl server_config = self._server.model_dump() if not self.config.sandbox_model_base_url and rollout_id and server_config["model_server_url"]: diff --git a/responses_api_agents/anyterminal_agent/tests/test_app.py b/responses_api_agents/anyterminal_agent/tests/test_app.py index d59bd5039c..73a47ea08c 100644 --- a/responses_api_agents/anyterminal_agent/tests/test_app.py +++ b/responses_api_agents/anyterminal_agent/tests/test_app.py @@ -269,7 +269,7 @@ def test_uses_per_task_timeout_by_default(self, tmp_path: Path) -> None: assert params.tb_agent_timeout == 900 def test_global_agent_timeout_overrides_per_task_timeout(self, tmp_path: Path) -> None: - agent = _make_setup_agent(tmp_path, global_agent_timeout=7200, tb_sandbox_ttl=12000) + agent = _make_setup_agent(tmp_path, global_agent_timeout=7200) body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path), "agent_timeout_sec": "900"}) params = agent._setup_params(body) @@ -277,19 +277,28 @@ def test_global_agent_timeout_overrides_per_task_timeout(self, tmp_path: Path) - assert params.tb_agent_timeout == 7200 def test_global_agent_timeout_applies_without_per_task_timeout(self, tmp_path: Path) -> None: - agent = _make_setup_agent(tmp_path, global_agent_timeout=7200, tb_sandbox_ttl=12000) + agent = _make_setup_agent(tmp_path, global_agent_timeout=7200) body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path)}) params = agent._setup_params(body) assert params.tb_agent_timeout == 7200 - def test_global_agent_timeout_exceeding_sandbox_ttl_is_rejected(self, tmp_path: Path) -> None: - agent = _make_setup_agent(tmp_path, global_agent_timeout=7200) - body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path)}) + def test_sandbox_ttl_is_derived_when_timeouts_would_outlive_it(self, tmp_path: Path) -> None: + agent = _make_setup_agent(tmp_path, global_agent_timeout=12500) + body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path), "verifier_timeout_sec": "900"}) + + params = agent._setup_params(body) + + assert params.tb_sandbox_ttl == 12500 + 900 + 600 + + def test_sandbox_ttl_default_kept_when_already_sufficient(self, tmp_path: Path) -> None: + agent = _make_setup_agent(tmp_path) + body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path), "agent_timeout_sec": "900"}) + + params = agent._setup_params(body) - with pytest.raises(ValueError, match="tb_sandbox_ttl"): - agent._setup_params(body) + assert params.tb_sandbox_ttl == 7200 def test_global_agent_timeout_rejects_zero(self) -> None: with pytest.raises(ValidationError, match="global_agent_timeout"):