feat: add global timeout to terminal bench

responses_api_agents/anyterminal_agent/app.py

@@ -352,6 +352,8 @@ class AnyTerminalAgentConfig(BaseResponsesAPIAgentConfig):
     sandbox_model_base_url: Optional[str] = None
     agent_runtime_source: str = "auto"
     tb_agent_timeout: int = 1800
+    # When set, overrides the per-task agent_timeout_sec from the dataset for every task.
+    global_agent_timeout: Optional[int] = Field(default=None, gt=0)
     tb_eval_timeout: int = 300
     tb_sandbox_ttl: int = 7200
     agent_overhead_mb: int = 2048  # extra container memory on top of the task's memory_mb for the
@@ -597,7 +599,7 @@ async def _run_agent(self, sandbox: AsyncSandbox, cfg: AnyTerminalInstanceConfig
         if result.return_code != 0:
             detail = result.stderr or result.stdout or ""
             print(f"[{cfg.task_name}] agent exit {result.return_code}: {detail[-2000:]}", flush=True)
-        return time.time() - t0, result.error_type == "timeout"
+        return time.time() - t0, result.error_type in ("timeout", "sandbox")
 
     async def _stage_tests(self, cfg: AnyTerminalInstanceConfig) -> None:
         """Copy the task's test files into the staging dir, visible to the sandbox at /tests."""
@@ -619,7 +621,7 @@ async def _run_eval(self, sandbox: AsyncSandbox, cfg: AnyTerminalInstanceConfig)
         result = await sandbox.exec(_apt_root_sandbox(cfg) + test_cmd, timeout_s=cfg.tb_eval_timeout, user="root")
         if result.return_code != 0:
             print(f"[{cfg.task_name}] eval exit {result.return_code}: {(result.stderr or '')[-2000:]}", flush=True)
-        return time.time() - t0, result.error_type == "timeout"
+        return time.time() - t0, result.error_type in ("timeout", "sandbox")
 
     async def process_single_datapoint(self) -> bool:
         cfg = self.config
@@ -824,13 +826,24 @@ def _setup_params(
 
         agent_run_id = f"{task_name}_{int(time.time())}_{uuid.uuid4().hex[:8]}"
 
-        # Per-task timeouts override config defaults when available.
+        # Per-task timeouts override config defaults when available, unless global_agent_timeout is set.
         config_overrides = {}
-        if problem_info.get("agent_timeout_sec"):
+        if self.config.global_agent_timeout is not None:
+            config_overrides["tb_agent_timeout"] = self.config.global_agent_timeout
+        elif problem_info.get("agent_timeout_sec"):
             config_overrides["tb_agent_timeout"] = int(float(problem_info["agent_timeout_sec"]))
         if problem_info.get("verifier_timeout_sec"):
             config_overrides["tb_eval_timeout"] = int(float(problem_info["verifier_timeout_sec"]))
 
+        # The container must outlive the task, or it gets torn down mid-run and the task scores as
+        # a real failure instead of the infra issue it is. Mirrors anyswe_agent's derivation
+        # (swebench_agent_timeout + swebench_tests_timeout + 600).
+        effective_agent_timeout = config_overrides.get("tb_agent_timeout", self.config.tb_agent_timeout)
+        effective_eval_timeout = config_overrides.get("tb_eval_timeout", self.config.tb_eval_timeout)
+        required_ttl = effective_agent_timeout + effective_eval_timeout + 600
+        if required_ttl > self.config.tb_sandbox_ttl:
+            config_overrides["tb_sandbox_ttl"] = required_ttl
+
         server_config = self._server.model_dump()
         if not self.config.sandbox_model_base_url and rollout_id and server_config["model_server_url"]:
             server_config["model_server_url"] = apply_rollout_prefix(server_config["model_server_url"], rollout_id)

responses_api_agents/anyterminal_agent/tests/test_app.py

@@ -27,21 +27,24 @@
 import time
 from pathlib import Path
 from types import SimpleNamespace
-from unittest.mock import AsyncMock, PropertyMock, patch
+from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch
 
 import pytest
+from pydantic import ValidationError
 
 from nemo_gym import PARENT_DIR
 from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming
 from nemo_gym.sandbox.providers.apptainer import ApptainerProvider
 from nemo_gym.sandbox.providers.apptainer import provider as apptainer_provider
 from nemo_gym.sandbox.providers.docker import DockerProvider
+from nemo_gym.server_utils import ServerClient
 from responses_api_agents.anyterminal_agent import app
 from responses_api_agents.anyterminal_agent.app import (
     _RUNNER_TEMPLATE,
     AnyTerminalAgent,
     AnyTerminalAgentConfig,
     AnyTerminalInstanceConfig,
+    AnyTerminalServerConfig,
     GymAgentHarnessProcessor,
     RunTerminalAgent,
     _build_provider,
@@ -167,8 +170,10 @@ def test_example_jsonl_parses(self) -> None:
 # ── helpers ───────────────────────────────────────────────────────────────────────
 
 
-def _make_body(content: str = "solve this") -> NeMoGymResponseCreateParamsNonStreaming:
-    return NeMoGymResponseCreateParamsNonStreaming(input=[{"role": "user", "content": content}], model="test-model")
+def _make_body(content: str = "solve this", **kwargs) -> NeMoGymResponseCreateParamsNonStreaming:
+    return NeMoGymResponseCreateParamsNonStreaming(
+        input=[{"role": "user", "content": content}], model="test-model", **kwargs
+    )
 
 
 def _make_instance_config(tmp_path: Path, **overrides) -> AnyTerminalInstanceConfig:
@@ -236,6 +241,70 @@ def test_toml_without_sections_returns_nones(self, tmp_path: Path) -> None:
         assert result.get("verifier_timeout_sec") is None
 
 
+# ── AnyTerminalAgent._setup_params ──────────────────────────────────────────────────
+
+
+def _make_setup_agent(tmp_path: Path, **config_overrides) -> AnyTerminalAgent:
+    # model_post_init has heavy side effects (deps install, provider resolution) that
+    # _setup_params doesn't touch, so bypass it and set only what _setup_params reads.
+    with patch.object(AnyTerminalAgent, "model_post_init", lambda self, context: None):
+        agent = AnyTerminalAgent(config=_config(**config_overrides), server_client=MagicMock(spec=ServerClient))
+    agent._server = AnyTerminalServerConfig(
+        run_session_id="test_session",
+        base_results_dir=tmp_path / "results",
+        model_server_url="",
+        nemo_gym_root=PARENT_DIR,
+        agent_deps_dir=tmp_path,
+    )
+    return agent
+
+
+class TestSetupParams:
+    def test_uses_per_task_timeout_by_default(self, tmp_path: Path) -> None:
+        agent = _make_setup_agent(tmp_path)
+        body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path), "agent_timeout_sec": "900"})
+
+        params = agent._setup_params(body)
+
+        assert params.tb_agent_timeout == 900
+
+    def test_global_agent_timeout_overrides_per_task_timeout(self, tmp_path: Path) -> None:
+        agent = _make_setup_agent(tmp_path, global_agent_timeout=7200)
+        body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path), "agent_timeout_sec": "900"})
+
+        params = agent._setup_params(body)
+
+        assert params.tb_agent_timeout == 7200
+
+    def test_global_agent_timeout_applies_without_per_task_timeout(self, tmp_path: Path) -> None:
+        agent = _make_setup_agent(tmp_path, global_agent_timeout=7200)
+        body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path)})
+
+        params = agent._setup_params(body)
+
+        assert params.tb_agent_timeout == 7200
+
+    def test_sandbox_ttl_is_derived_when_timeouts_would_outlive_it(self, tmp_path: Path) -> None:
+        agent = _make_setup_agent(tmp_path, global_agent_timeout=12500)
+        body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path), "verifier_timeout_sec": "900"})
+
+        params = agent._setup_params(body)
+
+        assert params.tb_sandbox_ttl == 12500 + 900 + 600
+
+    def test_sandbox_ttl_default_kept_when_already_sufficient(self, tmp_path: Path) -> None:
+        agent = _make_setup_agent(tmp_path)
+        body = _make_body(metadata={"task_name": "fix-git", "task_dir": str(tmp_path), "agent_timeout_sec": "900"})
+
+        params = agent._setup_params(body)
+
+        assert params.tb_sandbox_ttl == 7200
+
+    def test_global_agent_timeout_rejects_zero(self) -> None:
+        with pytest.raises(ValidationError, match="global_agent_timeout"):
+            _config(global_agent_timeout=0)
+
+
 # ── _instruction_from_input ───────────────────────────────────────────────────────
 
 
@@ -689,6 +758,25 @@ async def test_agent_timeout_sets_flag_and_masks(self, tmp_path: Path) -> None:
         assert metrics["agent_timed_out"] is True
         assert metrics["mask_sample"] is True
 
+    async def test_container_killed_mid_run_sets_flag_and_masks(self, tmp_path: Path) -> None:
+        # exec() on a container the TTL already removed returns error_type="sandbox" without
+        # raising, so this must be caught the same way a "timeout" is, not just via the
+        # except-block sandbox_failed path (which never fires here).
+        cfg = _make_instance_config(tmp_path)
+        sandbox = SimpleNamespace(
+            start=AsyncMock(),
+            exec=AsyncMock(return_value=_sandbox_result(return_code=125, error_type="sandbox")),
+            stop=AsyncMock(),
+        )
+        with patch("responses_api_agents.anyterminal_agent.app.AsyncSandbox", return_value=sandbox):
+            with patch.object(RunTerminalAgent, "_stage_tests", new=AsyncMock(return_value=None)):
+                await RunTerminalAgent(config=cfg).process_single_datapoint()
+
+        metrics = json.loads(cfg.metrics_fpath.read_text())
+        assert metrics["agent_timed_out"] is True
+        assert metrics["sandbox_failed"] is False
+        assert metrics["mask_sample"] is True
+
     async def test_sandbox_start_failure_is_isolated(self, tmp_path: Path) -> None:
         cfg = _make_instance_config(tmp_path)
         sandbox = SimpleNamespace(