diff --git a/src/benchflow/_utils/benchmark_repos.py b/src/benchflow/_utils/benchmark_repos.py index d583caa0..a7a1946a 100644 --- a/src/benchflow/_utils/benchmark_repos.py +++ b/src/benchflow/_utils/benchmark_repos.py @@ -47,19 +47,30 @@ class ResolvedSource: provenance: dict[str, Any] -def _repo_root() -> Path: - """Find the repo root via .git directory.""" +def _repo_root() -> Path | None: + """Find the enclosing git checkout root, or None outside one.""" d = Path.cwd() while d != d.parent: if (d / ".git").exists(): return d d = d.parent - return Path.cwd() + return None def _cache_dir() -> Path: - """Return the local cache directory for cloned dataset repos.""" - return _repo_root() / ".cache" / "datasets" + """Local cache directory for cloned dataset/catalog repos. + + Inside a git checkout: ``/.cache/datasets`` (the dev workflow). + Outside one: the user cache (``$XDG_CACHE_HOME``/``~/.cache``) — NEVER + the working directory, which a pip-installed ``bench`` would otherwise + pollute with multi-hundred-MB clones. + """ + root = _repo_root() + if root is not None: + return root / ".cache" / "datasets" + xdg = os.environ.get("XDG_CACHE_HOME") + base = Path(xdg) if xdg else Path.home() / ".cache" + return base / "benchflow" / "datasets" @contextmanager @@ -155,7 +166,7 @@ def _clone_repo_unlocked(org: str, repo: str, ref: str | None = None) -> Path: try: if clone_tmp.exists(): shutil.rmtree(clone_tmp) - cmd = ["git", "clone", "--depth", "1"] + cmd = ["git", "clone", "--quiet", "--depth", "1"] if ref and not _looks_like_commit_sha(ref): cmd.extend(["--branch", ref]) cmd.extend([url, str(clone_tmp)]) @@ -474,6 +485,9 @@ def infer_task_source_provenance(task_path: Path) -> dict[str, Any] | None: } repo_root = _repo_root() + if repo_root is None: + # Not inside a git checkout: no repo provenance to derive. + return None try: rel = task_resolved.relative_to(repo_root.resolve(strict=False)) except ValueError: diff --git a/src/benchflow/agents/registry.py b/src/benchflow/agents/registry.py index 8e2eedf2..f77beee2 100644 --- a/src/benchflow/agents/registry.py +++ b/src/benchflow/agents/registry.py @@ -1026,6 +1026,9 @@ def infer_env_key_for_model(model: str) -> str | None: AGENT_ALIASES: dict[str, str] = { + # mimo-code is the well-known vendor alias for the canonical mimo agent + # (the catalog's mimo-acp variant also claims it; local canonical wins). + "mimo-code": "mimo", "claude": "claude-agent-acp", "codex": "codex-acp", "gemini": "gemini", diff --git a/src/benchflow/agents/remote_manifests.py b/src/benchflow/agents/remote_manifests.py index 3335137b..e4a6d08d 100644 --- a/src/benchflow/agents/remote_manifests.py +++ b/src/benchflow/agents/remote_manifests.py @@ -65,9 +65,24 @@ def _source_root(spec: str) -> Path: if local.is_dir(): return local repo, _, ref = spec.partition("@") - from benchflow._utils.benchmark_repos import resolve_source + from benchflow._utils import benchmark_repos - return resolve_source(repo, ref=ref or None) + try: + return benchmark_repos.resolve_source(repo, ref=ref or None) + except Exception: + # Offline (or the refresh fetch failed): the cached clone is the + # catalog of record — a user who saw the full catalog online must + # not silently lose it, so fall back before giving up. + org, _, name = repo.partition("/") + cached = benchmark_repos._cache_dir() / org / name + if cached.is_dir(): + logger.warning( + "could not refresh agents source %s; using the cached catalog at %s", + spec, + cached, + ) + return cached + raise def _gap_fill( @@ -169,3 +184,74 @@ def _reset_for_tests() -> None: global _attempted, last_source_description _attempted = False last_source_description = "" + + +def fetch_one(name: str) -> bool: + """Fetch and register ONE catalog agent's manifest — never the full repo. + + Local-dir sources read ``acp//manifest.toml`` directly; remote + ``owner/repo[@ref]`` sources fetch that single file over HTTPS (raw + GitHub), so browsing/selecting an agent costs one small request instead + of a multi-hundred-MB clone. Local-wins semantics: a name already + registered is left untouched (returns True — the agent is available). + Returns False when the manifest cannot be fetched or is invalid. + """ + import os + + from benchflow.agents import registry + from benchflow.agents.manifest import load_agent_manifest + + if name in registry.AGENTS: + return True + spec = os.environ.get(AGENTS_SOURCE_ENV) or DEFAULT_AGENTS_SOURCE + if spec.strip().lower() in _OFF_VALUES: + return False + local = Path(spec).expanduser() + try: + if local.is_dir(): + text = (local / "acp" / name / "manifest.toml").read_text() + else: + import httpx + + repo, _, ref = spec.partition("@") + url = ( + f"https://raw.githubusercontent.com/{repo}/{ref or 'main'}" + f"/acp/{name}/manifest.toml" + ) + resp = httpx.get(url, timeout=30, follow_redirects=True) + if resp.status_code != 200: + logger.warning( + "catalog agent %r: manifest fetch returned HTTP %s (%s)", + name, + resp.status_code, + url, + ) + return False + text = resp.text + except Exception as exc: + logger.warning("catalog agent %r: manifest fetch failed: %s", name, exc) + return False + + import tempfile + + with tempfile.TemporaryDirectory() as td: + mpath = Path(td) / "manifest.toml" + mpath.write_text(text) + try: + loaded = load_agent_manifest(mpath) + except Exception as exc: + logger.warning("catalog agent %r: invalid manifest: %s", name, exc) + return False + from benchflow.agents.manifest import register_manifest_agents + + kept = _gap_fill([loaded], agents=registry.AGENTS, aliases=registry.AGENT_ALIASES) + if kept: + register_manifest_agents( + kept, + agents=registry.AGENTS, + aliases=registry.AGENT_ALIASES, + installers=registry.AGENT_INSTALLERS, + launch=registry.AGENT_LAUNCH, + ) + logger.info("catalog agent %r: registered from single-manifest fetch", name) + return name in registry.AGENTS diff --git a/src/benchflow/cli/init_cmd.py b/src/benchflow/cli/init_cmd.py new file mode 100644 index 00000000..0e00453c --- /dev/null +++ b/src/benchflow/cli/init_cmd.py @@ -0,0 +1,627 @@ +"""`bench init` (guided onboarding wizard) + `bench doctor` (health checks). + +Thin typer glue over :mod:`benchflow.onboarding`. Every prompt has a flag +mirror, so a fully-flagged invocation never blocks on stdin (CI mode); the +same check functions back both the wizard's closing smoke test and the +standalone doctor command. +""" + +from __future__ import annotations + +import contextlib +import os +import sys +from pathlib import Path + +import click +import typer + +# Line editing (arrows, backspace, history) for every visible prompt: +# click's prompts read via input(), which only edits when readline is loaded. +with contextlib.suppress(ImportError): # pragma: no cover — platform-dependent + import readline # noqa: F401 + +from benchflow import onboarding +from benchflow.cli._shared import console + +BENCHFLOW_HOME_ENV = "BENCHFLOW_HOME" + +# Concrete login commands for subscription-capable agents (wizard guidance). +_LOGIN_HINTS = { + "claude-agent-acp": "claude setup-token", + "codex-acp": "codex login", +} + + +def benchflow_home() -> Path: + return Path(os.environ.get(BENCHFLOW_HOME_ENV) or Path.home() / ".benchflow") + + +def _echo_results(results: list[onboarding.CheckResult]) -> bool: + all_ok = True + for r in results: + mark = "○" if r.skipped else ("✅" if r.ok else "❌") + typer.echo(f" {mark} {r.name}: {r.detail}") + all_ok &= r.ok + return all_ok + + +def _skip_note(results: list[onboarding.CheckResult]) -> str: + n = sum(1 for r in results if r.skipped) + return f" ({n} check(s) skipped — not verifiable before run time)" if n else "" + + +def _choose(title: str, options: list[tuple[str, str]], default: int | None = 1) -> int: + """Numbered menu: print options, return the chosen 1-based index. + + Enter accepts the default. Selection beats free-form typing for + discoverability; the ◆/│ step visuals follow the OpenClaw/clack wizard + look (rich degrades to plain text on non-tty). Free-text escape hatches + are modeled as an explicit "other" option by the caller. + """ + from rich.markup import escape + + console.print(f"\n[bold cyan]◆[/] [bold]{escape(title)}[/]") + for i, (label, desc) in enumerate(options, 1): + suffix = f" [dim]— {escape(desc)}[/]" if desc else "" + console.print(f"[dim]│[/] [cyan]{i})[/] {escape(label)}{suffix}") + # click.prompt (not typer.prompt): typer's re-prompt loop only catches + # its VENDORED click's UsageError, so a real click.IntRange failure + # escaped as a traceback. Real click's prompt re-prompts on typos. + return click.prompt("Select", type=click.IntRange(1, len(options)), default=default) + + +def _step(label: str, value: str) -> None: + """Confirmed-step rail line (the wizard's running summary).""" + from rich.markup import escape + + console.print(f"[dim]│[/] [green]●[/] {escape(label)} [bold]{escape(value)}[/]") + + +def _isatty() -> bool: + """Interactive-session check (module-level seam so tests can drive the + tty-only menus through CliRunner, which swaps sys.stdin).""" + return sys.stdin.isatty() + + +def _warn_shadow(auth_env: str, value: str) -> None: + """The shell export wins over anything init saves — say so.""" + exported = os.environ.get(auth_env) + if exported and exported != value: + typer.echo( + f"warning: {auth_env} is exported in your shell with a different" + " value — the exported variable will shadow this choice at run" + " time unless you unset it.", + err=True, + ) + + +def _fingerprint(value: str | None) -> str: + return f"…{value[-4:]}" if value and len(value) > 4 else "…" + + +def _auth_option(source: str, value: str | None, agent: str, auth_env: str): + if source == "subscription": + return ( + f"{agent}'s subscription login", + f"host login detected — no {auth_env} needed", + ) + if source == "environment": + return ( + f"{auth_env} from your environment ({_fingerprint(value)})", + "exported variable or saved setup", + ) + return ( + f"{auth_env} from {Path.cwd() / '.env'} ({_fingerprint(value)})", + "will be saved to your bench setup", + ) + + +def _save_setup_env( + home: Path, + *, + agent: str, + model: str, + updates: dict[str, str], +) -> None: + updates = onboarding.saved_setup_env_updates( + agent=agent, + model=model, + env=dict(os.environ), + updates=updates, + ) + onboarding.write_env_file(home / ".env", updates) + for key, value in updates.items(): + os.environ.setdefault(key, value) + + +def _wizard_auth_step(home: Path, agent: str, model: str, auth_env: str) -> None: + """OpenClaw-style auth step: every detected credential source is a listed + choice (subscription login included), manual entry is the escape hatch. + + Interactive (tty): a menu, defaulting to what the run path would use. + Non-interactive: the run-path-preferred source is taken automatically — + CI never blocks on stdin. + """ + sources = onboarding.detect_key_sources(auth_env, agent=agent) + use: tuple[str, str | None] | None = None + if sources and _isatty(): + options = [_auth_option(s, v, agent, auth_env) for s, v in sources] + options.append(("enter an API key", "typed hidden, stored for future runs")) + pick = _choose("Credentials:", options, default=1) + if pick <= len(sources): + use = sources[pick - 1] + elif sources: + use = sources[0] # each branch below prints its own line + if use is None: + key = typer.prompt(f"{auth_env}", hide_input=True) + _warn_shadow(auth_env, key) + _save_setup_env(home, agent=agent, model=model, updates={auth_env: key}) + os.environ[auth_env] = key # the explicit choice wins in-process + elif use[0] == "./.env": + value = use[1] or "" # detect_key_sources only lists ./.env with a value + _warn_shadow(auth_env, value) + _save_setup_env(home, agent=agent, model=model, updates={auth_env: value}) + os.environ[auth_env] = value # the explicit choice wins in-process + typer.echo( + f"✓ {auth_env} ({_fingerprint(value)}) from {Path.cwd() / '.env'} —" + f" saved to {home / '.env'}." + ) + elif use[0] == "environment": + typer.echo( + f"✓ Using {auth_env} from your environment ({_fingerprint(use[1])})." + ) + elif use[0] == "subscription": + if os.environ.get(auth_env): + # Honor the choice for this process (so the smoke verifies the + # subscription setup, not the declined key) and warn that the + # shell export will shadow it at run time. + typer.echo( + f"warning: {auth_env} is exported in your shell — it will" + " shadow the subscription login at run time unless you unset" + " it.", + err=True, + ) + os.environ.pop(auth_env, None) + typer.echo(f"✓ Using {agent}'s subscription login (no {auth_env} needed).") + + +def _osc52_copy(text: str) -> None: + """Best-effort clipboard copy via OSC52; harmless where unsupported.""" + import base64 + + if sys.stdout.isatty(): + payload = base64.b64encode(text.encode()).decode() + sys.stdout.write(f"\x1b]52;c;{payload}\x07") + sys.stdout.flush() + + +def register_init(app: typer.Typer) -> None: + @app.command("init", rich_help_panel="Core") + def init( + model: str = typer.Option( + None, "--model", "-m", help="Model id, e.g. deepseek/deepseek-v4-flash." + ), + agent: str = typer.Option(None, "--agent", "-a", help="Agent to run."), + dataset: str = typer.Option( + None, + "--dataset", + "-d", + help="Dataset spec (e.g. skillsbench@1.1) or a tasks dir path.", + ), + sandbox: str = typer.Option( + None, "--sandbox", help="Sandbox provider (docker, daytona, ...)" + ), + api_key: str = typer.Option( + None, + "--api-key", + help="Provider API key to store (falls back to subscription" + " login, then env var, then a hidden prompt).", + ), + skill_mode: str = typer.Option("with-skill", "--skill-mode"), + skip_smoke: bool = typer.Option( + False, "--skip-smoke", help="Skip the post-setup smoke test." + ), + full_smoke: bool = typer.Option( + False, + "--full-smoke", + help="Also run the credential-free oracle agent on one task in the" + " chosen sandbox (a real eval — takes minutes).", + ), + smoke_task: str = typer.Option( + None, "--smoke-task", help="Task name for --full-smoke's oracle run." + ), + ) -> None: + """Guided first-run setup: model → agent → tasks → sandbox → creds → smoke.""" + home = benchflow_home() + + if full_smoke and not smoke_task: + typer.echo("--full-smoke requires --smoke-task .", err=True) + raise typer.Exit(2) + + console.print("[bold cyan]◇ bench init[/] [dim]— first-run setup[/]") + + # Agent first (the thing the user came to benchmark). The menu lists + # locally-registered ACP agents only — populating it never touches the + # network. "other" (and the --agent flag) reach the full catalog, and + # the repo is cloned lazily there, only when an agent is actually + # resolved for a run. + if not agent: + agents = onboarding.acp_agents() + default = agents.index("pi-acp") + 1 if "pi-acp" in agents else 1 + options = [(a, "") for a in agents] + options.append( + ("other", "browse the full catalog (acp / ai-sdk / omnigent)") + ) + pick = _choose("Agent:", options, default=default) + if pick == len(options): + # Browse the STATIC catalog (zero network): path menu, then + # that path's agents. Only the selected agent is loaded — + # acp entries fetch ONE manifest; package paths (ai-sdk / + # omnigent) use the installed package or print the exact + # install command. + path_names = ("acp", "ai-sdk", "omnigent") + counts = {p: len(onboarding.path_choices(p)) for p in path_names} + ppick = _choose( + "Agent path:", + [(p, f"{counts[p]} agents") for p in path_names], + default=1, + ) + path = path_names[ppick - 1] + cat = onboarding.path_choices(path) + apick = _choose("Agent:", cat, default=1) + agent = cat[apick - 1][0] + from benchflow.agents.registry import AGENTS as _AGENTS_NOW + + if agent in _AGENTS_NOW: + pass # already available (installed package or core) + elif path == "acp": + console.print(f"[dim]│ fetching {agent} manifest…[/]") + from benchflow.agents import remote_manifests + + if not remote_manifests.fetch_one(agent): + typer.echo( + f"Could not fetch the manifest for {agent!r} —" + " check your network or BENCHFLOW_AGENTS_SOURCE.", + err=True, + ) + raise typer.Exit(1) + else: + hint = onboarding.install_hint(agent) + typer.echo( + f"{agent!r} comes from a pip package that is not" + " installed. Install it and re-run bench init:\n\n" + f" {hint}", + err=True, + ) + raise typer.Exit(1) + else: + agent = agents[pick - 1] + # Resolve like `bench eval run` does: aliases + the miss-driven catalog + # autoload (the only place the agents repo is cloned). A menu pick is + # already local, so this is a no-op for it; a typed/flagged catalog + # name resolves here. + from benchflow.agents.registry import resolve_agent + + try: + agent = resolve_agent(agent).name + except KeyError: + typer.echo(f"Unknown agent {agent!r} — see `bench agent list`.", err=True) + raise typer.Exit(1) from None + _step("agent", agent) + + # Subscription-capable agents: say IMMEDIATELY whether a local login + # was found, or guide the user to log in — don't make them discover + # it at the credentials step. + from benchflow.agents.registry import AGENTS as _AG + + _cfg_sub = _AG.get(agent) + _sa = _cfg_sub.subscription_auth if _cfg_sub else None + if _sa: + from benchflow.agents.env import check_subscription_auth + + # plain echo: these lines must survive narrow terminals and + # scripted greps unwrapped + if check_subscription_auth(agent, _sa.replaces_env): + typer.echo( + f"✓ subscription login found ({_sa.detect_file}) — no" + f" {_sa.replaces_env} needed." + ) + else: + hint = _LOGIN_HINTS.get(agent, "log in with the vendor CLI") + typer.echo( + f"○ no subscription login found ({_sa.detect_file}) — run" + f" `{hint}` to use your subscription, or continue with an" + f" {_sa.replaces_env}." + ) + + if not model: + from benchflow.agents.providers import PROVIDERS + from benchflow.agents.registry import AGENTS as _AGENTS + + names = onboarding.compatible_providers(agent) + _acfg = _AGENTS.get(agent) + _aproto = (_acfg.api_protocol or "") if _acfg else "" + + def _label(n: str) -> str: + cfg = PROVIDERS[n] + # a lone prefix equal to the provider name adds nothing + prefixes = [p for p in cfg.model_prefixes if p != n] + if prefixes: + return ", ".join(prefixes) + if not cfg.base_url: + return "BYO base URL" + # the endpoint this AGENT will use, not the provider's primary + return _aproto if _aproto in cfg.all_endpoints else cfg.api_protocol + + # Some agents have no registry endpoint but a first-class run + # path via a well-known inferred key: anthropic-native agents + # (subscription login / ANTHROPIC_API_KEY) and gemini (Google's + # native wire / GEMINI_API_KEY). Offer that as a synthetic entry, + # listed first + default, or those auth paths can never be + # reached from the menus. + synthetic = None # (label, desc, default model) + if _acfg and ( + ( + _acfg.subscription_auth + and _acfg.subscription_auth.replaces_env == "ANTHROPIC_API_KEY" + ) + or _aproto == "anthropic-messages" + ): + synthetic = ( + "anthropic", + "claude-* via subscription login or ANTHROPIC_API_KEY", + "claude-sonnet-4-6", + ) + elif agent == "gemini": + # gemini never routes through the provider registry — the + # 21-provider list would be a lie for it. + synthetic = ("google", "gemini-* via GEMINI_API_KEY", "gemini-3-pro") + names = [] + options = [(n, _label(n)) for n in names] + if synthetic: + options.insert(0, (synthetic[0], synthetic[1])) + options.append(("other", "type a full model id yourself")) + if synthetic: + default = 1 + elif "deepseek" in names: + default = names.index("deepseek") + 1 + elif "openai" in names: + default = names.index("openai") + 1 + else: + default = 1 + pick = _choose(f"Provider (routable by {agent}):", options, default=default) + if synthetic and pick == 1: + model = typer.prompt("Model id", default=synthetic[2]) + elif pick == len(options): # other + model = typer.prompt("Model id (provider/model or bare)") + else: + if synthetic: + pick -= 1 # past the synthetic entry + prov = PROVIDERS[names[pick - 1]] + catalog = [ + str(m.get("id") or m.get("name")) + for m in (prov.models or []) + if m.get("id") or m.get("name") + ] + if catalog: + mp = _choose( + f"Model ({names[pick - 1]}):", + [(m, "") for m in catalog] + [("other", "type a model id")], + ) + bare = ( + catalog[mp - 1] + if mp <= len(catalog) + else typer.prompt("Model id") + ) + else: + hint = ( + f" (e.g. {prov.model_prefixes[0]}-...)" + if prov.model_prefixes + else "" + ) + bare = typer.prompt(f"Model id{hint}") + model = bare if "/" in bare else f"{names[pick - 1]}/{bare}" + _step("model", model) + resolved = onboarding.resolve_provider(model) + if resolved: + prov_name, prov_cfg = resolved + auth_type, auth_env = prov_cfg.auth_type, prov_cfg.auth_env + else: + # Well-known model families (claude-*, gpt-*, gemini-*) run via + # their inferred key even without a registered provider endpoint. + from benchflow.agents.registry import infer_env_key_for_model + + inferred = infer_env_key_for_model(model) + if not inferred: + typer.echo(f"No registered provider recognizes {model!r}.", err=True) + raise typer.Exit(1) + prov_name, prov_cfg = None, None + auth_type, auth_env = "api_key", inferred + + # Consistency gate (also covers flag combinations): the run path + # would reject a protocol mismatch, so init must too. Native-wire + # agents (gemini) never route through the provider registry — for + # them the check is the model FAMILY, not the wire protocol. + if agent == "gemini": + from benchflow.agents.registry import infer_env_key_for_model + + if infer_env_key_for_model(model) != "GEMINI_API_KEY": + typer.echo( + f"Agent 'gemini' runs gemini-* models only; {model!r} is not one.", + err=True, + ) + raise typer.Exit(1) + offered = [agent] + else: + offered = onboarding.compatible_agents(model) + if agent not in offered: + typer.echo( + f"Agent {agent!r} cannot route {model!r} ({prov_name or 'provider'}" + " wire protocol mismatch) — the run would reject it. Compatible" + " agents:\n " + ", ".join(offered), + err=True, + ) + raise typer.Exit(1) + + if not dataset: + choices = onboarding.dataset_choices() + if choices: + opts = [*choices, ("a local tasks dir", "path to your own tasks")] + pick = _choose("Task set:", opts, default=1) + if pick == len(opts): + d = typer.prompt("Tasks dir path") + dataset = onboarding.normalize_dataset_input( + d, local_tasks_dir=True + ) + else: + dataset = choices[pick - 1][0] + else: + dataset = typer.prompt( + "Task set (dataset spec or tasks dir; registry unreachable)", + default="skillsbench@1.1", + ) + dataset = onboarding.normalize_dataset_input(dataset) + if "/" not in dataset and not dataset.startswith("."): + # Registry-style name: must parse as @ or the + # printed command will not run. + from benchflow._utils.dataset_registry import parse_dataset_spec + + try: + parse_dataset_spec(dataset) + except Exception as exc: + typer.echo( + f"Dataset {dataset!r} is not a valid spec: {exc}\n" + "Use @ (e.g. skillsbench@1.1) or a tasks" + " dir path.", + err=True, + ) + raise typer.Exit(1) from exc + _step("task set", dataset) + if not sandbox: + from benchflow.sandbox.providers import SANDBOX_PROVIDERS + + pick = _choose("Sandbox:", [(s, "") for s in SANDBOX_PROVIDERS]) + sandbox = SANDBOX_PROVIDERS[pick - 1] + _step("sandbox", sandbox) + + # Credentials: explicit --api-key > auto-detection (./.env in the + # working folder, then the environment incl. the saved setup, then + # subscription login) > hidden prompt as the last resort. Stored keys + # land in the private env file future runs auto-load. + if auth_type == "api_key" and auth_env: + try: + if api_key: + _warn_shadow(auth_env, api_key) + _save_setup_env( + home, + agent=agent, + model=model, + updates={auth_env: api_key}, + ) + os.environ.setdefault(auth_env, api_key) + else: + _wizard_auth_step(home, agent, model, auth_env) + except OSError as exc: + typer.echo( + f"Could not save credentials to {home / '.env'}: {exc}", + err=True, + ) + raise typer.Exit(1) from exc + elif api_key: + typer.echo( + f"--api-key is not used by provider {prov_name!r} (auth:" + f" {auth_type}) — configure {auth_type} credentials instead.", + err=True, + ) + raise typer.Exit(1) + + prefs = { + "agent": agent, + "model": model, + "dataset": dataset, + "sandbox": sandbox, + "skill_mode": skill_mode, + } + onboarding.save_prefs(home / "config.toml", prefs) + + if not skip_smoke: + if full_smoke and smoke_task: + # Stage 1 (Harbor's oracle pattern): prove install + sandbox + # plumbing with NO credentials involved. + import subprocess + + argv = onboarding.smoke_argv(prefs, task=smoke_task) + typer.echo( + "\nStage-1 smoke (oracle, no credentials): " + f"{onboarding.shell_join(argv)}" + ) + try: + oracle = subprocess.run(argv) + except FileNotFoundError as exc: + typer.echo( + "Cannot run the oracle smoke: `bench` is not on PATH" + f" ({exc}). Add your install's bin directory to PATH" + " and re-run, or use `bench doctor`.", + err=True, + ) + raise typer.Exit(1) from exc + mark = "✅" if oracle.returncode == 0 else "❌" + typer.echo(f" {mark} oracle sandbox run (rc={oracle.returncode})") + if oracle.returncode != 0: + typer.echo( + "\nSetup saved, but the stage-1 oracle smoke failed —" + " the sandbox plumbing is broken independent of your" + " credentials.", + err=True, + ) + raise typer.Exit(1) + typer.echo("\nSmoke test:") + env = dict(os.environ) + if api_key and auth_env: + # Verify the key that was just SAVED, not whatever happened to + # be exported before init ran. + env[auth_env] = api_key + results = onboarding.run_doctor(model, sandbox, env, agent=agent) + ok = _echo_results(results) + if ok and _skip_note(results): + typer.echo(f"Smoke passed{_skip_note(results)}.") + if not ok: + typer.echo( + "\nSetup saved, but the smoke test failed — fix the ❌ rows" + " above and re-check with `bench doctor`.", + err=True, + ) + raise typer.Exit(1) + + cmd = onboarding.final_command(prefs) + console.print("\n[bold green]◆ Ready.[/] Run your first eval with:\n") + typer.echo(f" {cmd}\n") + _osc52_copy(cmd) + + @app.command("doctor", rich_help_panel="Core") + def doctor() -> None: + """Re-validate the saved setup: sandbox, provider key, LiteLLM route, + model ping.""" + home = benchflow_home() + try: + prefs = onboarding.load_prefs(home / "config.toml") + except Exception: + prefs = None # corrupt TOML — same remediation as missing keys + if not prefs or not {"model", "sandbox"} <= prefs.keys(): + typer.echo( + "Saved setup is missing or incomplete — run `bench init` first.", + err=True, + ) + raise typer.Exit(1) + onboarding.load_env_file(home / ".env") + results = onboarding.run_doctor( + prefs["model"], + prefs["sandbox"], + dict(os.environ), + agent=prefs.get("agent"), + ) + ok = _echo_results(results) + note = _skip_note(results) + typer.echo(f"\nAll checks passed.{note}" if ok else "\nSome checks failed.") + raise typer.Exit(0 if ok else 1) diff --git a/src/benchflow/cli/main.py b/src/benchflow/cli/main.py index bb6ef131..26f2c9e4 100644 --- a/src/benchflow/cli/main.py +++ b/src/benchflow/cli/main.py @@ -49,6 +49,7 @@ from benchflow.cli.environment import register_environment from benchflow.cli.eval_artifacts import postprocess_eval_artifacts, run_matrix_eval from benchflow.cli.hub import register_hub +from benchflow.cli.init_cmd import register_init from benchflow.cli.monitor import register_monitor from benchflow.cli.sandbox import register_sandbox from benchflow.cli.skills import register_skills @@ -110,6 +111,12 @@ def _cli_main( ] = None, ) -> None: """The universal environment framework — run, author, and adopt agent benchmarks.""" + # Credentials saved by `bench init` fill env gaps for every subcommand; + # anything already exported wins (setdefault semantics). + from benchflow import onboarding + from benchflow.cli.init_cmd import benchflow_home + + onboarding.load_env_file(benchflow_home() / ".env") def _parse_agent_env(entries: list[str] | None) -> dict[str, str]: @@ -1218,6 +1225,7 @@ def eval_view( register_train(app) register_hub(app) register_agent(app) +register_init(app) register_adopt_deprecated(app) register_sandbox(app) register_environment(app) diff --git a/src/benchflow/onboarding.py b/src/benchflow/onboarding.py new file mode 100644 index 00000000..ede3322d --- /dev/null +++ b/src/benchflow/onboarding.py @@ -0,0 +1,746 @@ +"""Onboarding logic behind ``bench init`` / ``bench doctor``. + +The logic the CLI wizard is thin glue over: a private env file for +credentials (secrets live in env vars, per the provider registry's +``auth_env`` contract), TOML preferences, provider/agent compatibility +filtering, and the health checks the post-init smoke test and ``bench +doctor`` share. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from benchflow.agents.providers import ProviderConfig + + +def _parse_env_line(line: str) -> tuple[str, str] | None: + """One dotenv line -> (key, value), or None for comments/malformed.""" + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + return None + if line.startswith("export "): + line = line[len("export ") :].lstrip() + key, _, raw = line.partition("=") + key = key.strip() + if not key or any(c.isspace() for c in key): + return None + value = raw.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + return key, value + + +def write_env_file(path: str | Path, updates: dict[str, str]) -> None: + """Merge *updates* into the env file at *path* (created, mode forced 0600). + + The merge is line-preserving: keys already present are replaced in place, + new keys are appended, and everything the parser does not understand — + comments, malformed lines — is kept verbatim, so a hand-edited file + survives the next init. An undecodable file raises OSError instead of + being clobbered. + """ + path = Path(path) + try: + lines = path.read_text().splitlines() + except FileNotFoundError: + lines = [] + except UnicodeDecodeError as exc: + raise OSError( + f"refusing to rewrite {path}: not valid UTF-8 ({exc}); fix or" + " delete it first" + ) from exc + remaining = dict(updates) + out: list[str] = [] + for line in lines: + parsed = _parse_env_line(line) + if parsed and parsed[0] in remaining: + out.append(f'{parsed[0]}="{remaining.pop(parsed[0])}"') + else: + out.append(line) + out.extend(f'{k}="{v}"' for k, v in remaining.items()) + path.parent.mkdir(parents=True, exist_ok=True) + path.touch(mode=0o600, exist_ok=True) + path.chmod(0o600) + path.write_text("\n".join(out) + "\n") + + +def read_env_file(path: str | Path) -> dict[str, str]: + """Parse a dotenv-style file; missing/unreadable file reads as empty. + + Tolerates the dialects people hand-paste: ``export KEY=...`` prefixes and + single- or double-quoted values. Malformed lines (empty or whitespace + keys) and undecodable bytes are skipped/degraded — never fatal, because + the CLI auto-loads this file on every invocation, including the ones + needed to repair it. + """ + path = Path(path) + try: + text = path.read_text() + except FileNotFoundError: + return {} + except (OSError, UnicodeDecodeError) as exc: + import sys + + print(f"warning: cannot read {path}: {exc}", file=sys.stderr) + return {} + values: dict[str, str] = {} + for line in text.splitlines(): + parsed = _parse_env_line(line) + if parsed: + values[parsed[0]] = parsed[1] + return values + + +def load_env_file(path: str | Path) -> list[str]: + """setdefault every env-file entry into os.environ; report what was set. + + The real environment always wins — the file only fills gaps, so an + exported key or a CI secret is never clobbered by a stale init. + """ + import os + + applied = [] + for key, value in read_env_file(path).items(): + if key not in os.environ: + os.environ[key] = value + applied.append(key) + return applied + + +def save_prefs(path: str | Path, prefs: dict) -> None: + """Write run preferences (agent/model/dataset/sandbox...) as TOML.""" + import tomli_w + + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(tomli_w.dumps(prefs).encode()) + + +def load_prefs(path: str | Path) -> dict: + """Read preferences TOML; missing file reads as empty.""" + import tomllib + + path = Path(path) + if not path.exists(): + return {} + return tomllib.loads(path.read_text()) + + +def resolve_provider(model: str) -> tuple[str, ProviderConfig] | None: + """Map a model id ("provider/model" or bare) to (provider_name, config). + + Returns None when no registered provider claims the model — the wizard + then falls back to the model's inferred well-known key + (claude-*/gpt-*/gemini-* via infer_env_key_for_model), or errors if none + can be inferred. + """ + from benchflow.agents.providers import PROVIDERS, find_provider_for_bare_model + + if "/" in model: + prefix = model.split("/", 1)[0] + cfg = PROVIDERS.get(prefix) + return (prefix, cfg) if cfg else None + return find_provider_for_bare_model(model) + + +def compatible_agents(model: str | None = None) -> list[str]: + """Registered agent names that can actually route *model*. + + With no model (the agent-first wizard flow) the full registered list is + returned, minus the never-routed agents; the protocol filter then applies + in the other direction (compatible_providers). + + Reuses the same provider-protocol gate the run path enforces + (env._provider_supports_agent_protocol), so the wizard never offers an + agent the run would immediately reject — e.g. anthropic-messages or + openai-responses agents on an openai-completions-only provider. Agents + that bypass provider routing entirely are never offered (gemini speaks + its provider's native wire; oracle is not a registered agent — special- + cased in rollout — and is listed defensively). + """ + from benchflow.agents.env import _provider_supports_agent_protocol + from benchflow.agents.registry import AGENTS + + non_routed = {"oracle", "gemini"} + resolved = resolve_provider(model) if model else None + names = [] + for name, cfg in sorted(AGENTS.items()): + if name in non_routed: + continue + if resolved and not _provider_supports_agent_protocol( + resolved[1], cfg.api_protocol or "" + ): + continue + names.append(name) + return names + + +from dataclasses import dataclass # noqa: E402 + + +@dataclass(frozen=True) +class CheckResult: + """One doctor/smoke row: what was checked, verdict, human detail. + + ``skipped=True`` marks a check that could not be exercised (e.g. run-time + auth) — it never fails a doctor run (``ok`` stays True) but is rendered + distinctly and counted in the summary so green does not over-certify. + """ + + name: str + ok: bool + detail: str = "" + skipped: bool = False + + +def _sanitize(text: str) -> str: + """Strip control characters so server-supplied text cannot inject + terminal escape sequences through doctor/smoke output.""" + return "".join(ch for ch in text if ch.isprintable() or ch in " \t") + + +def shell_join(argv: list[str]) -> str: + """Render argv as a copy-pasteable shell command.""" + import shlex + + return shlex.join(argv) + + +def normalize_dataset_input( + dataset: str, + *, + local_tasks_dir: bool = False, + cwd: str | Path | None = None, +) -> str: + """Normalize task-dir values without changing valid registry specs.""" + if "/" in dataset or dataset.startswith("."): + return dataset + if local_tasks_dir or (Path(cwd or ".") / dataset).is_dir(): + return f"./{dataset}" + return dataset + + +def _ping_headers(prov_name: str, protocol: str, key: str) -> dict[str, str]: + if protocol == "openai-completions": + if prov_name.startswith("azure-foundry-"): + return {"api-key": key} + return {"Authorization": f"Bearer {key}"} + if protocol == "anthropic-messages": + # x-api-key is the Anthropic wire header (anthropic-version is + # required by that wire contract); Azure's Anthropic surface accepts + # api-key too, so send both key headers. + return { + "x-api-key": key, + "api-key": key, + "anthropic-version": "2023-06-01", + } + return {} + + +def _openai_completion_token_limit(bare_model: str) -> dict[str, int]: + """Return the supported minimal token limit for chat-completions pings.""" + if bare_model.startswith(("gpt-5", "o1", "o3", "o4")): + return {"max_completion_tokens": 8} + return {"max_tokens": 1} + + +def _run_path_env(model: str, env: dict[str, str], agent: str | None) -> dict[str, str]: + """Normalize a check env with the same resolver used before real runs.""" + if not agent: + return dict(env) + from benchflow.agents.env import resolve_agent_env + + try: + return resolve_agent_env(agent, model, env) + except Exception: + return dict(env) + + +def saved_setup_env_updates( + *, + agent: str, + model: str, + env: dict[str, str], + updates: dict[str, str], +) -> dict[str, str]: + """Return env-file updates, including derived provider setup values.""" + saved = dict(updates) + resolved = resolve_provider(model) + if not resolved or not resolved[0].startswith("azure-foundry-"): + return saved + normalized = _run_path_env(model, {**env, **updates}, agent) + resource = normalized.get("AZURE_RESOURCE") + if resource: + saved.setdefault("AZURE_RESOURCE", resource) + return saved + + +def model_ping(model: str, env: dict[str, str], transport=None) -> CheckResult: + """Verify key + model id + endpoint with one minimal completion. + + GET /models is not used: it can 200 while the actual route is broken + (wrong model id, upstream 5xx) — a minimal completion is the cheapest + request that exercises the full path. The endpoint uses the same + resolve_base_url + path-segment join as the run path, but prefers the + openai-completions endpoint when a provider has several — it validates + the key/route, not necessarily the endpoint the chosen agent will use. + Provider classes the ping cannot exercise (ADC, Bedrock bearer auth) are + skipped honestly instead of reported as failures. + """ + import httpx + + from benchflow.agents.providers import resolve_base_url, strip_provider_prefix + + resolved = resolve_provider(model) + if not resolved: + from benchflow.agents.registry import infer_env_key_for_model + + if infer_env_key_for_model(model): + # Well-known family (claude-*/gpt-*/gemini-*) with no registered + # endpoint: nothing to ping — the run wires it natively. + return CheckResult( + "model ping", + True, + "skipped — no registered endpoint to ping for this model" + " family; auth is exercised at run time", + skipped=True, + ) + return CheckResult("model ping", False, f"no registered provider for {model!r}") + prov_name, cfg = resolved + name = f"model ping ({prov_name})" + if cfg.auth_type != "api_key": + return CheckResult( + name, + True, + f"skipped — {cfg.auth_type} auth is exercised at run time", + skipped=True, + ) + key = env.get(cfg.auth_env or "", "") + if not key: + return CheckResult(name, False, f"{cfg.auth_env} is not set") + + endpoints = cfg.all_endpoints + bare_model = strip_provider_prefix(model) + if "openai-completions" in endpoints: + protocol = "openai-completions" + path, ok_field = "/chat/completions", "choices" + headers = _ping_headers(prov_name, protocol, key) + payload = { + "model": bare_model, + "messages": [{"role": "user", "content": "ping"}], + **_openai_completion_token_limit(bare_model), + } + elif "anthropic-messages" in endpoints: + protocol = "anthropic-messages" + path, ok_field = "/v1/messages", "content" + headers = _ping_headers(prov_name, protocol, key) + payload = { + "model": bare_model, + "messages": [{"role": "user", "content": "ping"}], + "max_tokens": 1, + } + else: + return CheckResult( + name, + True, + f"skipped — no pingable endpoint ({sorted(endpoints)})", + skipped=True, + ) + try: + base = resolve_base_url(cfg, env, protocol=protocol).rstrip("/") + except KeyError as exc: + return CheckResult(name, False, _sanitize(str(exc))) + url = f"{base}{path}" + import logging + + _httpx_logger = logging.getLogger("httpx") + _prev_level = _httpx_logger.level + _httpx_logger.setLevel(logging.WARNING) # keep request logs out of the rows + try: + with httpx.Client(transport=transport, timeout=30) as client: + resp = client.post(url, json=payload, headers=headers) + except httpx.HTTPError as exc: + return CheckResult(name, False, _sanitize(f"request failed: {exc}")) + finally: + _httpx_logger.setLevel(_prev_level) + if resp.status_code == 200: + try: + body = resp.json() + except ValueError: + body = None + if isinstance(body, dict) and ok_field in body: + return CheckResult(name, True, f"1-token completion OK ({url})") + return CheckResult( + name, + False, + _sanitize(f"200 but not a completion response: {resp.text[:200]}"), + ) + return CheckResult( + name, False, _sanitize(f"HTTP {resp.status_code}: {resp.text[:200]}") + ) + + +def run_doctor( + model: str, + sandbox: str, + env: dict[str, str], + ping_transport=None, + skip_ping: bool = False, + agent: str | None = None, +) -> list[CheckResult]: + """The shared check set behind `bench doctor` and the init smoke stage. + + When *agent* is given and a host subscription login covers the model's + key (check_subscription_auth), the key/route/ping rows are skipped — a + subscription setup must not be failed by checks that only understand API + keys. + """ + env = _run_path_env(model, env, agent) + results: list[CheckResult] = [] + if sandbox == "docker": + from benchflow.sandbox.docker import DockerSandbox + + try: + DockerSandbox.preflight() + except SystemExit as exc: + results.append(CheckResult("docker", False, _sanitize(str(exc)))) + else: + results.append(CheckResult("docker", True, "docker daemon ready")) + elif sandbox == "daytona": + has = bool(env.get("DAYTONA_API_KEY")) + results.append( + CheckResult( + "daytona (DAYTONA_API_KEY)", + has, + "set" if has else "DAYTONA_API_KEY is not set", + ) + ) + else: + from benchflow.sandbox.providers import SANDBOX_PROVIDERS + + known = sandbox in SANDBOX_PROVIDERS + results.append( + CheckResult( + "sandbox", + known, + f"{sandbox} (no init-time checks for this provider)" + if known + else f"unknown sandbox {sandbox!r} (expected one of" + f" {', '.join(SANDBOX_PROVIDERS)})", + ) + ) + resolved = resolve_provider(model) + auth_env: str | None = None + if resolved: + _, cfg = resolved + if cfg.auth_type == "api_key": + auth_env = cfg.auth_env + else: + # No registered provider endpoint, but well-known model families + # (claude-*, gpt-*, gemini-*) still run via their inferred key. + from benchflow.agents.registry import infer_env_key_for_model + + auth_env = infer_env_key_for_model(model) + if not auth_env: + results.append( + CheckResult("provider", False, f"no registered provider for {model!r}") + ) + return results + + if agent and auth_env and not env.get(auth_env): + from benchflow.agents.env import check_subscription_auth + + if check_subscription_auth(agent, auth_env): + results.append( + CheckResult( + f"provider auth ({agent})", + True, + "skipped — host subscription login covers this model;" + " key/route/ping checks do not apply", + skipped=True, + ) + ) + return results + + if auth_env: + has = bool(env.get(auth_env)) + results.append( + CheckResult( + f"provider key ({auth_env})", + has, + "set" if has else f"{auth_env} is not set", + ) + ) + # LiteLLM route resolution is pure (no network) and catches the + # env/route errors the proxy would hit at run time — a doctor-green ping + # alone does not prove the proxy lane resolves. + try: + from benchflow.providers.litellm_config import resolve_litellm_route + + route = resolve_litellm_route(model, env) + results.append( + CheckResult("litellm route", True, f"resolves (alias {route.model_alias})") + ) + except Exception as exc: + results.append( + CheckResult( + "litellm route", False, _sanitize(f"{type(exc).__name__}: {exc}") + ) + ) + if not skip_ping: + results.append(model_ping(model, env, transport=ping_transport)) + return results + + +def _run_args(prefs: dict, *, agent: str, model: str | None) -> list[str]: + args = ["bench", "eval", "run", "--agent", agent] + if model: + args += ["--model", model] + dataset = prefs["dataset"] + if "/" in dataset or dataset.startswith("."): + args += ["--tasks-dir", dataset] + else: + args += ["-d", dataset] + args += ["--sandbox", prefs["sandbox"]] + if prefs.get("skill_mode"): + args += ["--skill-mode", prefs["skill_mode"]] + return args + + +def final_command(prefs: dict) -> str: + """The ready-to-run command the wizard prints (and copies) at the end.""" + return shell_join(_run_args(prefs, agent=prefs["agent"], model=prefs["model"])) + + +def smoke_argv(prefs: dict, task: str) -> list[str]: + """Stage-1 smoke: the credential-free oracle agent on ONE task — proves + install + sandbox plumbing before any API key is involved.""" + return [*_run_args(prefs, agent="oracle", model=None), "--include", task] + + +def detect_key_sources( + auth_env: str, agent: str | None = None, cwd: str | Path | None = None +) -> list[tuple[str, str | None]]: + """Every credential source found for *auth_env*, in run-path order. + + Order matches what the RUN would actually use (resolve_agent_env + inherits a local .env first, then the process environment, and only uses + host subscription auth when no key source is present): ``./.env`` in the + working folder, then the process environment (which already includes the + saved ~/.benchflow/.env via startup autoload), then host subscription + login (needs *agent*; value None — nothing to store). The wizard's auth + menu shows all of them; the non-interactive path takes the first. + """ + import os + + sources: list[tuple[str, str | None]] = [] + value = read_env_file(Path(cwd or ".") / ".env").get(auth_env) + if value: + sources.append(("./.env", value)) + if os.environ.get(auth_env): + sources.append(("environment", os.environ[auth_env])) + if agent: + from benchflow.agents.env import check_subscription_auth + + if check_subscription_auth(agent, auth_env): + sources.append(("subscription", None)) + return sources + + +def detect_key( + auth_env: str, agent: str | None = None, cwd: str | Path | None = None +) -> tuple[str | None, str | None]: + """First (run-path-preferred) credential source, or ``(None, None)``.""" + sources = detect_key_sources(auth_env, agent=agent, cwd=cwd) + return sources[0] if sources else (None, None) + + +def dataset_choices() -> list[tuple[str, str]]: + """(spec, description) picker entries from the dataset registry, newest + version first within each name; empty when the registry is unreachable + (the wizard then falls back to free-form entry).""" + from benchflow._utils import dataset_registry as dr + + try: + entries = [ + e + for e in dr.load_registry(dr.DEFAULT_REGISTRY_SOURCE) + if isinstance(e, dict) + ] + except Exception: + return [] + + def _vkey(e): + try: + return tuple(int(p) for p in str(e.get("version", "")).split(".")) + except ValueError: + return () + + ordered = sorted(entries, key=_vkey, reverse=True) + ordered.sort(key=lambda e: str(e.get("name", ""))) + import textwrap + + return [ + ( + f"{e['name']}@{e['version']}", + textwrap.shorten(str(e.get("description", "")), width=80, placeholder="…"), + ) + for e in ordered + if e.get("name") and e.get("version") + ] + + +def compatible_providers(agent: str) -> list[str]: + """Registered provider names the chosen agent can route — the same + protocol gate as compatible_agents, applied in the other direction for + the agent-first wizard flow.""" + from benchflow.agents.env import _provider_supports_agent_protocol + from benchflow.agents.providers import PROVIDERS + from benchflow.agents.registry import AGENTS + + cfg = AGENTS.get(agent) + protocol = (cfg.api_protocol or "") if cfg else "" + return [ + name + for name in sorted(PROVIDERS) + if _provider_supports_agent_protocol(PROVIDERS[name], protocol) + ] + + +# The ACP catalog names in benchflow-ai/agents (acp//manifest.toml). +# STATIC on purpose: browsing must cost zero network; selecting an entry +# lazily fetches only that agent's manifest (remote_manifests.fetch_one). A +# stale entry fails its fetch with a clear message — rot is loud, not silent. +CATALOG_AGENTS: tuple[str, ...] = ( + "amp-acp", + "auggie", + "autohand", + "cline", + "codebuddy-code", + "cortex-code", + "corust-agent", + "crow-cli", + "dimcode", + "dirac", + "factory-droid", + "fast-agent", + "github-copilot-cli", + "glm-acp-agent", + "goose", + "grok-build", + "junie", + "kilo", + "mimo-acp", + "minion-code", + "mistral-vibe", + "nova", + "poolside", + "qoder", + "qwen-code", + "sigit", + "stakpak", + "vtcode", +) + + +# Package-path agents (pip-installed, not manifest-fetchable). Static like +# CATALOG_AGENTS; selecting one that is not installed yields the exact +# install command (install_hint) instead of a dead end. +AI_SDK_AGENTS: tuple[str, ...] = ( + "ai-sdk", + "ai-sdk-pi", + "ai-sdk-codex", + "ai-sdk-claude-code", + "ai-sdk-deepagents", + "ai-sdk-opencode", +) +OMNIGENT_AGENTS: tuple[str, ...] = ( + "omnigent-pi", + "omnigent-claude", + "omnigent-openai-agents", + "omnigent-codex", + "omnigent-goose", + "omnigent-qwen", + "omnigent-kimi", + "omnigent-hermes", + "omnigent-copilot", + "omnigent-cursor", + "omnigent-antigravity", + "omnigent-claude-native", + "omnigent-codex-native", + "omnigent-goose-native", + "omnigent-qwen-native", + "omnigent-kimi-native", + "omnigent-hermes-native", + "omnigent-cursor-native", + "omnigent-antigravity-native", + "omnigent-kiro-native", + "omnigent-opencode-native", + "omnigent-pi-native", +) +_AI_SDK_SUBDIRS = { + "ai-sdk": "ai-sdk/acp", + "ai-sdk-pi": "ai-sdk/harness-pi", + "ai-sdk-codex": "ai-sdk/harness-codex", + "ai-sdk-claude-code": "ai-sdk/harness-claude-code", + "ai-sdk-deepagents": "ai-sdk/harness-deepagents", + "ai-sdk-opencode": "ai-sdk/harness-opencode", +} + + +def path_choices(path: str) -> list[tuple[str, str]]: + """Static (name, desc) entries for one adaptation path's browse menu.""" + from benchflow.agents.registry import AGENTS + + names = { + "acp": CATALOG_AGENTS, + "ai-sdk": AI_SDK_AGENTS, + "omnigent": OMNIGENT_AGENTS, + }[path] + return [(n, "" if n not in AGENTS else "installed") for n in names] + + +def install_hint(name: str) -> str | None: + """The exact install command for a package-path agent, or None for + manifest-path (acp) agents.""" + base = "https://github.com/benchflow-ai/agents@main" + if name.startswith("omnigent-"): + return f'uv pip install "omnigent-benchflow @ git+{base}#subdirectory=omnigent"' + sub = _AI_SDK_SUBDIRS.get(name) + if sub: + return f'uv pip install "{name} @ git+{base}#subdirectory={sub}"' + return None + + +def catalog_choices() -> list[tuple[str, str]]: + """Static (name, description) entries for the wizard's catalog browse — + ACP catalog agents not already registered locally. Zero network.""" + from benchflow.agents.registry import AGENTS + + return [(n, "") for n in CATALOG_AGENTS if n not in AGENTS] + + +def acp_agents() -> list[str]: + """Locally-registered ACP-native agents for the wizard menu — NO network. + + Only what is already in the registry (core built-ins + any installed + plugin packages), minus the other adaptation paths (ai-sdk-* / omnigent-*) + which the wizard does not list. The full benchflow-ai/agents manifest + catalog is fetched lazily — only when an agent is actually resolved for a + run (resolve_agent's miss path), not to populate a menu — so a bare + ``bench init`` never clones the repo. Names not shown here stay reachable + via ``--agent `` and the menu's "other" escape, both of which + resolve (and lazily fetch) on demand. + """ + return [ + name + for name in compatible_agents() + if not ( + name == "ai-sdk" + or name.startswith("ai-sdk-") + or name.startswith("omnigent-") + ) + ] diff --git a/tests/init_cli_helpers.py b/tests/init_cli_helpers.py new file mode 100644 index 00000000..21d0d82f --- /dev/null +++ b/tests/init_cli_helpers.py @@ -0,0 +1,29 @@ +"""Shared helpers for bench init CLI tests.""" + +from __future__ import annotations + +from typer.testing import CliRunner + +from benchflow.cli.main import app + +__all__ = ["_init_args", "app", "runner"] + +runner = CliRunner() + + +def _init_args(home, extra=()): + return [ + "init", + "--model", + "deepseek/deepseek-v4-flash", + "--agent", + "pi-acp", + "--dataset", + "skillsbench@1.1", + "--sandbox", + "docker", + "--api-key", + "sk-test-123", + "--skip-smoke", + *extra, + ] diff --git a/tests/test_daytona_dind_compose_up.py b/tests/test_daytona_dind_compose_up.py index a1103515..5711b5e5 100644 --- a/tests/test_daytona_dind_compose_up.py +++ b/tests/test_daytona_dind_compose_up.py @@ -165,7 +165,10 @@ def test_long_exec_heartbeat_returns_promptly_after_fast_command(): capture_output=True, check=True, text=True, - timeout=2, + # "Promptly" = far below the 60s heartbeat interval; 2s flaked on + # loaded CI runners (bash -l startup + subshell spawn), so give + # headroom without weakening the assertion. + timeout=15, ) assert result.stdout == "done" diff --git a/tests/test_init_cli.py b/tests/test_init_cli.py new file mode 100644 index 00000000..58c9f989 --- /dev/null +++ b/tests/test_init_cli.py @@ -0,0 +1,429 @@ +"""`bench init` / `bench doctor` non-interactive, doctor, and smoke behavior.""" + +from __future__ import annotations + +from tests.init_cli_helpers import _init_args, app, runner + + +def test_non_interactive_writes_files_and_prints_command(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + result = runner.invoke(app, _init_args(tmp_path)) + assert result.exit_code == 0, result.output + # secrets → private env file; prefs → config; final command printed + from benchflow import onboarding + + assert onboarding.read_env_file(tmp_path / ".env") == { + "DEEPSEEK_API_KEY": "sk-test-123" + } + assert ((tmp_path / ".env").stat().st_mode & 0o777) == 0o600 + prefs = onboarding.load_prefs(tmp_path / "config.toml") + assert prefs["agent"] == "pi-acp" + assert ( + "bench eval run --agent pi-acp --model deepseek/deepseek-v4-flash" + in result.output + ) + + +def test_non_interactive_azure_init_persists_derived_resource(tmp_path, monkeypatch): + """Guards PR #883: saved Azure setup includes run-path-derived resource.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.setenv( + "AZURE_API_ENDPOINT", "https://example-resource.openai.azure.com/" + ) + monkeypatch.delenv("AZURE_RESOURCE", raising=False) + monkeypatch.setattr("benchflow.agents.env.load_dotenv_env", lambda: {}) + + result = runner.invoke( + app, + [ + "init", + "--model", + "azure-foundry-openai/gpt-5.5", + "--agent", + "pi-acp", + "--dataset", + "skillsbench@1.1", + "--sandbox", + "docker", + "--api-key", + "sk-az", + "--skip-smoke", + ], + ) + + assert result.exit_code == 0, result.output + from benchflow import onboarding + + assert onboarding.read_env_file(tmp_path / ".env") == { + "AZURE_API_KEY": "sk-az", + "AZURE_RESOURCE": "example-resource", + } + + +def test_incompatible_agent_for_model_is_rejected(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + result = runner.invoke( + app, + [ + "init", + "--model", + "deepseek/deepseek-v4-flash", + "--agent", + "codex-acp", # openai-responses wire: the run path would reject it + "--dataset", + "skillsbench@1.1", + "--sandbox", + "docker", + "--api-key", + "sk-x", + "--skip-smoke", + ], + ) + assert result.exit_code != 0 + assert "codex-acp" in result.output and "deepseek" in result.output + + +def test_doctor_without_setup_hints_at_init(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 1 + assert "bench init" in result.output + + +def test_doctor_reports_rows_and_fails_on_broken_setup(tmp_path, monkeypatch): + """Saved setup but no key and no docker: every row shows, exit code 1.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) + monkeypatch.setattr("shutil.which", lambda _: None) + from benchflow import onboarding + + onboarding.save_prefs( + tmp_path / "config.toml", + { + "agent": "pi-acp", + "model": "deepseek/deepseek-v4-flash", + "dataset": "skillsbench", + "sandbox": "docker", + }, + ) + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 1 + assert "docker" in result.output + assert "DEEPSEEK_API_KEY" in result.output + assert "❌" in result.output + + +def test_interactive_wizard_prompts_and_completes(tmp_path, monkeypatch): + """No flags: the wizard walks numbered menus (provider → model → agent → + dataset → sandbox), auto-detects credentials, and only prompts for the + key when nothing is found.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) + monkeypatch.chdir(tmp_path) # no ./.env here -> key prompt is the fallback + monkeypatch.setattr( + "benchflow.onboarding.dataset_choices", + lambda: [("skillsbench@1.1", "87-task benchmark")], + ) + answers = "\n".join( + [ + "", # agent menu -> default (pi-acp) + "", # provider menu (filtered) -> default (deepseek) + "deepseek-v4-flash", # model id (deepseek has no catalog) + "", # dataset menu -> default (skillsbench@1.1) + "", # sandbox menu -> default (docker) + "sk-wizard-key", # hidden api key (nothing auto-detected) + ] + ) + result = runner.invoke(app, ["init", "--skip-smoke"], input=answers + "\n") + assert result.exit_code == 0, result.output + from benchflow import onboarding + + assert onboarding.read_env_file(tmp_path / ".env") == { + "DEEPSEEK_API_KEY": "sk-wizard-key" + } + assert "bench eval run --agent pi-acp" in result.output + + +def test_startup_autoloads_saved_env_file(tmp_path, monkeypatch): + """A key stored by a previous init is visible to later invocations + without exporting anything (the CLI auto-loads ~/.benchflow/.env).""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) + from benchflow import onboarding + + onboarding.write_env_file(tmp_path / ".env", {"DEEPSEEK_API_KEY": "sk-saved"}) + result = runner.invoke(app, [*_init_args(tmp_path)[:-3], "--skip-smoke"]) + # (same init args minus --api-key: the saved key must be found) + assert result.exit_code == 0, result.output + assert "DEEPSEEK_API_KEY from your environment" in result.output + + +def test_full_smoke_runs_credential_free_oracle_stage(tmp_path, monkeypatch): + """--full-smoke executes the stage-1 oracle eval (sandbox plumbing proof) + before the credential checks; the exact argv is the assembled smoke + command.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + calls = [] + + def fake_run(argv, **kwargs): + calls.append(argv) + + class R: + returncode = 0 + + return R() + + from benchflow.onboarding import CheckResult + + monkeypatch.setattr("subprocess.run", fake_run) + monkeypatch.setattr( + "benchflow.onboarding.run_doctor", + lambda *a, **k: [CheckResult("stub", True, "")], + ) + result = runner.invoke( + app, + [*_init_args(tmp_path)[:-1], "--full-smoke", "--smoke-task", "citation-check"], + ) + assert result.exit_code == 0, result.output + # doctor checks still run (docker/key/ping will fail here — that's fine, + # exit code reflects them); the oracle stage must have been invoked first. + assert any( + argv[:5] == ["bench", "eval", "run", "--agent", "oracle"] for argv in calls + ), result.output + assert any("citation-check" in argv for argv in calls) + + +def test_full_smoke_oracle_failure_fails_init(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + + def failing_run(argv, **kwargs): + class R: + returncode = 1 + + return R() + + monkeypatch.setattr("subprocess.run", failing_run) + result = runner.invoke( + app, + [*_init_args(tmp_path)[:-1], "--full-smoke", "--smoke-task", "citation-check"], + ) + assert result.exit_code == 1 + assert "oracle" in result.output + + +def test_full_smoke_without_task_errors_loudly(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + result = runner.invoke(app, [*_init_args(tmp_path), "--full-smoke"]) + assert result.exit_code == 2 + assert "--smoke-task" in result.output + + +def test_api_key_flag_rejected_for_non_api_key_provider(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + result = runner.invoke( + app, + [ + "init", + "--model", + "google-vertex/gemini-3-pro", + "--agent", + "pi-acp", + "--dataset", + "skillsbench@1.1", + "--sandbox", + "docker", + "--api-key", + "sk-x", + "--skip-smoke", + ], + ) + assert result.exit_code != 0 + assert "adc" in result.output.lower() + + +def test_smoke_verifies_the_key_just_provided(tmp_path, monkeypatch): + """--api-key must be what the checks verify, even if a different key is + already exported (verify what was saved, not what happened to be set).""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-stale-exported") + seen = {} + + def fake_doctor(model, sandbox, env, **kw): + seen["key"] = env.get("DEEPSEEK_API_KEY") + from benchflow.onboarding import CheckResult + + return [CheckResult("stub", True, "")] + + monkeypatch.setattr("benchflow.onboarding.run_doctor", fake_doctor) + result = runner.invoke(app, _init_args(tmp_path)[:-1]) # drop --skip-smoke + assert result.exit_code == 0, result.output + assert seen["key"] == "sk-test-123" + + +def test_bare_model_with_inferable_key_completes(tmp_path, monkeypatch): + """No registered provider, but a well-known key family (claude-*): the + wizard proceeds, stores the key under the inferred env var, and skips the + endpoint ping honestly.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + result = runner.invoke( + app, + [ + "init", + "--model", + "claude-opus-4-6", + "--agent", + "claude-agent-acp", + "--dataset", + "skillsbench@1.1", + "--sandbox", + "docker", + "--api-key", + "sk-ant-x", + "--skip-smoke", + ], + ) + assert result.exit_code == 0, result.output + from benchflow import onboarding + + assert onboarding.read_env_file(tmp_path / ".env") == { + "ANTHROPIC_API_KEY": "sk-ant-x" + } + + +def test_subscription_login_skips_key_prompt(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) + monkeypatch.setattr( + "benchflow.agents.env.check_subscription_auth", lambda agent, key: True + ) + # no --api-key, no env key, no stdin: would block on the hidden prompt + # unless the subscription path answers first. + result = runner.invoke(app, [*_init_args(tmp_path)[:-3], "--skip-smoke"]) + assert result.exit_code == 0, result.output + assert "subscription" in result.output.lower() + + +def test_default_dataset_produces_a_parseable_spec(tmp_path, monkeypatch): + """The wizard's whole job is handing over a command that RUNS: a + registry-style dataset must carry a version (name@version) or init must + reject it — bare 'skillsbench' fails bench eval run's dataset parsing.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + result = runner.invoke( + app, + [ + "init", + "--model", + "deepseek/deepseek-v4-flash", + "--agent", + "pi-acp", + "--dataset", + "skillsbench", # version-less registry name + "--sandbox", + "docker", + "--api-key", + "sk-x", + "--skip-smoke", + ], + ) + assert result.exit_code != 0 + assert "@" in result.output # points at name@version + + result = runner.invoke( + app, + [ + "init", + "--model", + "deepseek/deepseek-v4-flash", + "--agent", + "pi-acp", + "--dataset", + "skillsbench@1.1", + "--sandbox", + "docker", + "--api-key", + "sk-x", + "--skip-smoke", + ], + ) + assert result.exit_code == 0, result.output + assert "-d skillsbench@1.1" in result.output + from benchflow._utils.dataset_registry import parse_dataset_spec + + parse_dataset_spec("skillsbench@1.1") # the emitted value must parse + + +def test_doctor_survives_corrupt_and_partial_config(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + (tmp_path / "config.toml").write_text('model = "unterminated') + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 1 + assert "bench init" in result.output + + (tmp_path / "config.toml").write_text('agent = "pi-acp"\n') # missing keys + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 1 + assert "bench init" in result.output + + +def test_full_smoke_missing_bench_binary_fails_cleanly(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + + def gone(argv, **kw): + raise FileNotFoundError(2, "No such file", "bench") + + monkeypatch.setattr("subprocess.run", gone) + result = runner.invoke( + app, + [*_init_args(tmp_path)[:-1], "--full-smoke", "--smoke-task", "t1"], + ) + assert result.exit_code == 1 + assert "PATH" in result.output # clear message, not a traceback + + +def test_full_smoke_prints_shell_quoted_command(tmp_path, monkeypatch): + """Guards PR #883: stage-1 smoke output is copy-pasteable.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path / "home")) + tasks_dir = tmp_path / "my tasks" + tasks_dir.mkdir() + calls = [] + + def fake_run(argv, **kwargs): + calls.append(argv) + + class R: + returncode = 0 + + return R() + + from benchflow.onboarding import CheckResult + + monkeypatch.setattr("subprocess.run", fake_run) + monkeypatch.setattr( + "benchflow.onboarding.run_doctor", + lambda *a, **k: [CheckResult("stub", True, "")], + ) + + result = runner.invoke( + app, + [ + "init", + "--model", + "deepseek/deepseek-v4-flash", + "--agent", + "pi-acp", + "--dataset", + str(tasks_dir), + "--sandbox", + "docker", + "--api-key", + "sk-x", + "--full-smoke", + "--smoke-task", + "citation-check", + ], + ) + + assert result.exit_code == 0, result.output + assert f"--tasks-dir '{tasks_dir}'" in result.output + assert calls[0][calls[0].index("--tasks-dir") + 1] == str(tasks_dir) diff --git a/tests/test_init_cli_catalog_agents.py b/tests/test_init_cli_catalog_agents.py new file mode 100644 index 00000000..3f660238 --- /dev/null +++ b/tests/test_init_cli_catalog_agents.py @@ -0,0 +1,195 @@ +"""`bench init` catalog-agent and package-path menu behavior.""" + +from __future__ import annotations + +from tests.init_cli_helpers import app, runner + + +def _manifest_source(tmp_path, monkeypatch, name="probe-flag-agent"): + from benchflow.agents import remote_manifests + + d = tmp_path / "src" / name + d.mkdir(parents=True) + (d / "manifest.toml").write_text( + f'contract_version = "1.0"\nname = "{name}"\nprotocol = "acp"\n' + 'install_cmd = "true"\nlaunch_cmd = "true"\n' + ) + monkeypatch.setenv(remote_manifests.AGENTS_SOURCE_ENV, str(tmp_path / "src")) + remote_manifests._reset_for_tests() + return name + + +def test_agent_flag_reaches_catalog_agents_via_autoload(tmp_path, monkeypatch): + """--agent naming a catalog-only agent must work like `bench run` does + (the miss path autoloads) — not exit with a bogus protocol mismatch.""" + name = _manifest_source(tmp_path, monkeypatch) + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path / "home")) + try: + result = runner.invoke( + app, + [ + "init", + "--agent", + name, + "--model", + "deepseek/deepseek-v4-flash", + "--dataset", + "skillsbench@1.1", + "--sandbox", + "docker", + "--api-key", + "sk-x", + "--skip-smoke", + ], + ) + assert result.exit_code == 0, result.output + finally: + from benchflow.agents import registry, remote_manifests + + remote_manifests._reset_for_tests() + registry.AGENTS.pop(name, None) + registry.AGENT_INSTALLERS.pop(name, None) + registry.AGENT_LAUNCH.pop(name, None) + + +def test_unknown_agent_flag_says_unknown_not_protocol_mismatch(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + from benchflow.agents import remote_manifests + + monkeypatch.setenv(remote_manifests.AGENTS_SOURCE_ENV, "off") + remote_manifests._reset_for_tests() + result = runner.invoke( + app, + [ + "init", + "--agent", + "definitely-not-an-agent-9000", + "--model", + "deepseek/deepseek-v4-flash", + "--dataset", + "skillsbench@1.1", + "--sandbox", + "docker", + "--api-key", + "sk-x", + "--skip-smoke", + ], + ) + assert result.exit_code == 1 + assert "Unknown agent" in result.output + assert "protocol mismatch" not in result.output + + +def test_other_agent_offers_static_catalog_and_fetches_one_manifest( + tmp_path, monkeypatch +): + """'other' shows a STATIC catalog list (zero network) and selecting an + entry fetches ONLY that agent's manifest — never the full repo.""" + from benchflow.agents import registry, remote_manifests + + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path / "home")) + monkeypatch.chdir(tmp_path) + # hermetic single-manifest source + from benchflow import onboarding + + # Deterministic target regardless of suite order: earlier tests (e.g. + # manifest parity) may have registered the WHOLE catalog — evict one + # entry for the duration of this test and restore it afterwards. + target = onboarding.CATALOG_AGENTS[0] + saved = ( + registry.AGENTS.pop(target, None), + registry.AGENT_INSTALLERS.pop(target, None), + registry.AGENT_LAUNCH.pop(target, None), + ) + d = tmp_path / "src" / "acp" / target + d.mkdir(parents=True) + (d / "manifest.toml").write_text( + f'contract_version = "1.0"\nname = "{target}"\nprotocol = "acp"\n' + 'api_protocol = "openai-completions"\n' + 'install_cmd = "true"\nlaunch_cmd = "true"\n' + ) + monkeypatch.setenv(remote_manifests.AGENTS_SOURCE_ENV, str(tmp_path / "src")) + # full-repo loading must NOT happen + monkeypatch.setattr( + remote_manifests, + "autoload_remote_manifest_agents", + lambda: (_ for _ in ()).throw(AssertionError("full catalog load!")), + ) + monkeypatch.setattr( + "benchflow.onboarding.dataset_choices", lambda: [("skillsbench@1.1", "")] + ) + n_local = len(onboarding.acp_agents()) + target_idx = [n for n, _ in onboarding.path_choices("acp")].index(target) + 1 + answers = "\n".join( + [ + str(n_local + 1), # agent menu: "other" -> path menu + "1", # path menu: acp + str(target_idx), # pick it from the static list + "", # provider -> deepseek + "deepseek-v4-flash", + "", # dataset + "", # sandbox + "sk-k", # key + ] + ) + try: + result = runner.invoke(app, ["init", "--skip-smoke"], input=answers + "\n") + assert result.exit_code == 0, result.output + assert f"--agent {target}" in result.output + finally: + registry.AGENTS.pop(target, None) + registry.AGENT_INSTALLERS.pop(target, None) + registry.AGENT_LAUNCH.pop(target, None) + for store, val in zip( + (registry.AGENTS, registry.AGENT_INSTALLERS, registry.AGENT_LAUNCH), + saved, + strict=True, + ): + if val is not None: + store[target] = val + + +def test_other_shows_three_paths_then_that_paths_agents(tmp_path, monkeypatch): + """'other' -> path menu (acp / ai-sdk / omnigent with counts) -> that + path's static agent list. Package paths guide installation instead of + dead-ending.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + from benchflow import onboarding + + n_local = len(onboarding.acp_agents()) + answers = "\n".join( + [ + str(n_local + 1), # agent menu -> other + "3", # path menu -> omnigent + "1", # first omnigent agent (package not installed here? may be) + ] + ) + result = runner.invoke(app, ["init", "--skip-smoke"], input=answers + "\n") + out = result.output + assert "acp" in out and "ai-sdk" in out and "omnigent" in out # path menu + assert "omnigent-" in out # that path's agents listed + + +def test_uninstalled_package_agent_gets_install_guidance(tmp_path, monkeypatch): + """Selecting a package-path agent that is not importable prints the exact + install command instead of a crash or a dead end.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + from benchflow.agents import registry + + # simulate not-installed: evict ai-sdk-codex if some env registered it + saved = registry.AGENTS.pop("ai-sdk-codex", None) + try: + from benchflow import onboarding + + n_local = len(onboarding.acp_agents()) + idx = [n for n, _ in onboarding.path_choices("ai-sdk")].index("ai-sdk-codex") + answers = "\n".join([str(n_local + 1), "2", str(idx + 1)]) + result = runner.invoke(app, ["init", "--skip-smoke"], input=answers + "\n") + assert result.exit_code == 1 + assert "uv pip install" in result.output + assert "ai-sdk/harness-codex" in result.output # the exact subdirectory + finally: + if saved is not None: + registry.AGENTS["ai-sdk-codex"] = saved diff --git a/tests/test_init_cli_interactive.py b/tests/test_init_cli_interactive.py new file mode 100644 index 00000000..4fa261e9 --- /dev/null +++ b/tests/test_init_cli_interactive.py @@ -0,0 +1,474 @@ +"""`bench init` interactive menu and credential-source behavior.""" + +from __future__ import annotations + +from tests.init_cli_helpers import _init_args, app, runner + + +def test_stale_exported_key_shadow_is_warned(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-stale-exported") + result = runner.invoke(app, _init_args(tmp_path)) + assert result.exit_code == 0, result.output + assert "shadow" in result.output.lower() + + +def test_wizard_is_selection_driven_with_auto_key_detection(tmp_path, monkeypatch): + """Hermes-style UX: every step is a numbered menu (or Enter for the + default) — and the key is auto-detected from ./.env in the working + folder, so the user never types it.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path / "home")) + monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) + monkeypatch.chdir(tmp_path) + (tmp_path / ".env").write_text('DEEPSEEK_API_KEY="sk-from-folder"\n') + monkeypatch.setattr( + "benchflow.onboarding.dataset_choices", + lambda: [("skillsbench@1.1", "87-task benchmark")], + ) + answers = "\n".join( + [ + "", # agent menu -> Enter = default (pi-acp) + "", # provider menu (filtered to pi-acp-routable) -> Enter = deepseek + "deepseek-v4-flash", # model (free text w/ hint; deepseek has no catalog) + "", # dataset menu -> Enter = default (skillsbench@1.1) + "", # sandbox menu -> Enter = default (docker) + ] + ) + result = runner.invoke(app, ["init", "--skip-smoke"], input=answers + "\n") + assert result.exit_code == 0, result.output + # told the user the absolute source, a key fingerprint, and the destination + assert str(tmp_path / ".env") in result.output + assert "…" in result.output and "saved" in result.output + from benchflow import onboarding + + # passed through into the saved setup for future runs + assert onboarding.read_env_file(tmp_path / "home" / ".env") == { + "DEEPSEEK_API_KEY": "sk-from-folder" + } + assert ( + "bench eval run --agent pi-acp --model deepseek/deepseek-v4-flash" + in result.output + ) + # menus were shown, not free-text demands + assert "1)" in result.output + + +def test_provider_menu_is_filtered_by_chosen_agent(tmp_path, monkeypatch): + """Agent comes first; the provider menu then only offers providers that + agent can route (codex-acp speaks openai-responses -> deepseek, which is + completions-only, must not be offered).""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + # pick codex-acp by name from the agent menu, then abort at the provider + # menu with an out-of-range answer + EOF; the menu text is what we assert. + result = runner.invoke( + app, + ["init", "--skip-smoke", "--agent", "codex-acp"], + input="\n", # provider menu: Enter default, then model prompt EOFs + ) + menu = result.output.split("Provider", 1)[-1] + menu = menu.split("Select", 1)[0] + assert "openai" in menu + assert "deepseek" not in menu + + +def test_local_tasks_dir_bare_name_is_normalized_not_rejected(tmp_path, monkeypatch): + """Picking 'a local tasks dir' and answering a bare relative name must + produce a --tasks-dir command, not a registry-spec rejection.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path / "home")) + monkeypatch.chdir(tmp_path) + (tmp_path / "mytasks").mkdir() + monkeypatch.setattr( + "benchflow.onboarding.dataset_choices", + lambda: [("skillsbench@1.1", "")], + ) + answers = "\n".join( + [ + "", # agent -> pi-acp + "", # provider -> deepseek + "deepseek-v4-flash", + "2", # dataset menu: "a local tasks dir" + "mytasks", # bare relative name + "", # sandbox -> docker + "sk-k", # key prompt + ] + ) + result = runner.invoke(app, ["init", "--skip-smoke"], input=answers + "\n") + assert result.exit_code == 0, result.output + assert "--tasks-dir ./mytasks" in result.output + + +def test_flagged_local_tasks_dir_bare_name_is_normalized(tmp_path, monkeypatch): + """Guards PR #883: fully flagged local task dirs mirror the prompt path.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path / "home")) + monkeypatch.chdir(tmp_path) + (tmp_path / "mytasks").mkdir() + + result = runner.invoke( + app, + [ + "init", + "--model", + "deepseek/deepseek-v4-flash", + "--agent", + "pi-acp", + "--dataset", + "mytasks", + "--sandbox", + "docker", + "--api-key", + "sk-x", + "--skip-smoke", + ], + ) + + assert result.exit_code == 0, result.output + assert "--tasks-dir ./mytasks" in result.output + + +def test_auto_key_detection_prefers_cwd_dotenv_over_exported_key(tmp_path, monkeypatch): + """Guards PR #883: init validates the key the final run will use.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path / "home")) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-exported") + (tmp_path / ".env").write_text('DEEPSEEK_API_KEY="sk-from-cwd"\n') + + result = runner.invoke( + app, + [ + "init", + "--model", + "deepseek/deepseek-v4-flash", + "--agent", + "pi-acp", + "--dataset", + "skillsbench@1.1", + "--sandbox", + "docker", + "--skip-smoke", + ], + ) + + assert result.exit_code == 0, result.output + from benchflow import onboarding + + assert onboarding.read_env_file(tmp_path / "home" / ".env") == { + "DEEPSEEK_API_KEY": "sk-from-cwd" + } + assert str(tmp_path / ".env") in result.output + assert "shadow" in result.output.lower() + + +def test_provider_labels_show_the_matched_protocol(tmp_path, monkeypatch): + """A provider's label must show the endpoint the CHOSEN agent will use — + aws-bedrock's primary wire is openai-responses, but next to + claude-agent-acp it serves anthropic-messages and must say so.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + result = runner.invoke( + app, ["init", "--skip-smoke", "--agent", "claude-agent-acp"], input="\n" + ) + menu = result.output.split("Provider", 1)[-1].split("Select", 1)[0] + assert "openai-responses" not in menu + assert "anthropic-messages" in menu + assert "BYO" in menu # vllm: no canonical URL, caller supplies semantics + + +def test_interactive_auth_menu_lists_subscription_as_a_choice(tmp_path, monkeypatch): + """OpenClaw-style auth step: when a subscription login AND a key are both + available, the user chooses — subscription is a listed option, not a + silent auto-decision.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-exported-key") + monkeypatch.setattr( + "benchflow.agents.env.check_subscription_auth", lambda a, k: True + ) + monkeypatch.setattr("benchflow.cli.init_cmd._isatty", lambda: True) + answers = "\n".join( + [ + "", # agent -> pi-acp + "", # provider -> deepseek + "deepseek-v4-flash", + "", # dataset (registry stubbed below) + "", # sandbox -> docker + "2", # auth menu: pick the subscription login + ] + ) + monkeypatch.setattr( + "benchflow.onboarding.dataset_choices", lambda: [("skillsbench@1.1", "")] + ) + result = runner.invoke(app, ["init", "--skip-smoke"], input=answers + "\n") + assert result.exit_code == 0, result.output + assert "subscription" in result.output.lower() # listed as an option + assert "…-key" in result.output or "…" in result.output # key fingerprinted + + +def test_subscription_pick_is_honored_by_the_smoke(tmp_path, monkeypatch): + """Choosing the subscription login while a key is exported must make the + smoke verify the SUBSCRIPTION setup (key rows skipped) and warn that the + shell export will shadow it at run time.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-exported") + monkeypatch.setattr( + "benchflow.agents.env.check_subscription_auth", lambda a, k: True + ) + monkeypatch.setattr("benchflow.cli.init_cmd._isatty", lambda: True) + seen = {} + + def fake_doctor(model, sandbox, env, **kw): + seen["key_in_env"] = "DEEPSEEK_API_KEY" in env + from benchflow.onboarding import CheckResult + + return [CheckResult("stub", True, "")] + + monkeypatch.setattr("benchflow.onboarding.run_doctor", fake_doctor) + result = runner.invoke( + app, + [ + "init", + "--agent", + "pi-acp", + "--model", + "deepseek/deepseek-v4-flash", + "--dataset", + "skillsbench@1.1", + "--sandbox", + "docker", + ], + input="2\n", # credentials menu: 1=env key, 2=subscription + ) + assert result.exit_code == 0, result.output + assert seen["key_in_env"] is False # the declined key is NOT verified + assert "shadow" in result.output.lower() # told about the shell export + + +def test_claude_agent_gets_native_anthropic_provider_and_subscription( + tmp_path, monkeypatch +): + """An anthropic-native agent (claude-agent-acp) must offer 'anthropic' + in the provider menu — the registry has no such endpoint, but the run + path supports it via subscription login / ANTHROPIC_API_KEY — and picking + it must surface the subscription login in the credentials menu.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.setattr( + "benchflow.agents.env.check_subscription_auth", lambda a, k: True + ) + monkeypatch.setattr("benchflow.cli.init_cmd._isatty", lambda: True) + monkeypatch.setattr( + "benchflow.onboarding.dataset_choices", lambda: [("skillsbench@1.1", "")] + ) + answers = "\n".join( + [ + "1", # provider menu: anthropic (native) listed first for this agent + "", # model id -> default (claude-sonnet-4-6) + "", # dataset -> default + "", # sandbox -> docker + "1", # credentials menu: subscription login (listed!) + ] + ) + result = runner.invoke( + app, + ["init", "--skip-smoke", "--agent", "claude-agent-acp"], + input=answers + "\n", + ) + assert result.exit_code == 0, result.output + menu = result.output.split("Provider", 1)[-1] + assert "anthropic" in menu.split("Select", 1)[0] + assert "subscription login" in result.output + assert "--model claude-sonnet-4-6" in result.output + + +def test_subscription_status_announced_right_after_agent_choice(tmp_path, monkeypatch): + """Choosing a subscription-capable agent immediately reports whether a + local login was found — or guides the user to log in.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + # found case + monkeypatch.setattr( + "benchflow.agents.env.check_subscription_auth", lambda a, k: True + ) + result = runner.invoke( + app, ["init", "--skip-smoke", "--agent", "claude-agent-acp"], input="\n" + ) + assert "subscription login found" in result.output.lower() + # not-found case -> login guidance + monkeypatch.setattr( + "benchflow.agents.env.check_subscription_auth", lambda a, k: False + ) + result = runner.invoke( + app, ["init", "--skip-smoke", "--agent", "claude-agent-acp"], input="\n" + ) + out = result.output.lower() + assert "no" in out and "subscription login" in out + assert "claude setup-token" in result.output # concrete login guidance + + +def test_codex_subscription_guidance_uses_codex_login(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + "benchflow.agents.env.check_subscription_auth", lambda a, k: False + ) + result = runner.invoke( + app, ["init", "--skip-smoke", "--agent", "codex-acp"], input="\n" + ) + assert "codex login" in result.output + + +def test_gemini_flags_path_works_via_inferred_key(tmp_path, monkeypatch): + """gemini is menu-excluded (native wire) but --agent gemini with a + gemini-* model must work via the inferred GEMINI_API_KEY — not die with a + bogus 'wire protocol mismatch'.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + result = runner.invoke( + app, + [ + "init", + "--agent", + "gemini", + "--model", + "gemini-3-pro", + "--dataset", + "skillsbench@1.1", + "--sandbox", + "docker", + "--api-key", + "sk-fake", + "--skip-smoke", + ], + ) + assert result.exit_code == 0, result.output + assert "--agent gemini --model gemini-3-pro" in result.output + from benchflow import onboarding + + assert onboarding.read_env_file(tmp_path / ".env") == {"GEMINI_API_KEY": "sk-fake"} + + +def test_gemini_with_wrong_family_model_gets_clear_error(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + result = runner.invoke( + app, + [ + "init", + "--agent", + "gemini", + "--model", + "deepseek/deepseek-v4-flash", + "--dataset", + "skillsbench@1.1", + "--sandbox", + "docker", + "--api-key", + "sk-fake", + "--skip-smoke", + ], + ) + assert result.exit_code == 1 + assert "gemini" in result.output and "gemini-*" in result.output + assert "protocol mismatch" not in result.output # no misleading diagnosis + + +def test_gemini_interactive_gets_native_provider_menu(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + result = runner.invoke( + app, ["init", "--skip-smoke", "--agent", "gemini"], input="\n" + ) + menu = result.output.split("Provider", 1)[-1].split("Select", 1)[0] + assert "google" in menu and "GEMINI_API_KEY" in menu + assert "deepseek" not in menu # not the 21-provider lie + + +def test_codex_provider_default_is_openai_not_bedrock(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + result = runner.invoke( + app, ["init", "--skip-smoke", "--agent", "codex-acp"], input="\n" + ) + # the Select prompt default index must point at openai + menu = result.output.split("Provider", 1)[-1] + import re + + m = re.search(r"(\d+)\) openai ", menu) + d = re.search(r"Select \[(\d+)\]", menu) + assert m and d and m.group(1) == d.group(1), menu[:400] + + +def test_provider_label_not_a_duplicate_of_the_name(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + result = runner.invoke( + app, ["init", "--skip-smoke", "--agent", "deepagents"], input="\n" + ) + menu = result.output.split("Provider", 1)[-1].split("Select", 1)[0] + assert "deepseek — deepseek" not in menu # no information-free labels + + +def test_prompts_have_line_editing(): + """Arrows/backspace must work in every prompt: click's input() only gets + line editing when readline is loaded — importing the wizard module must + load it (guarded for platforms without it).""" + import sys + + import benchflow.cli.init_cmd # noqa: F401 + + assert "readline" in sys.modules + + +def test_menu_reprompts_on_non_integer_input(tmp_path, monkeypatch): + """Typos at a Select menu must re-prompt, not crash: typer.prompt's + re-prompt loop catches its VENDORED click's UsageError, so a real + click.IntRange BadParameter escaped as a traceback (auth-matrix find).""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + answers = "\n".join( + [ + "not-a-number", # agent menu: garbage -> must re-prompt + "", # then Enter = default pi-acp + "", # provider -> deepseek + "deepseek-v4-flash", + "skillsbench@1.1", + "docker", + "sk-k", + ] + ) + # dataset/sandbox prompts appear as menus; feed by index-agnostic strings + monkeypatch.setattr( + "benchflow.onboarding.dataset_choices", lambda: [("skillsbench@1.1", "")] + ) + answers = "\n".join(["not-a-number", "", "", "deepseek-v4-flash", "", "", "sk-k"]) + result = runner.invoke(app, ["init", "--skip-smoke"], input=answers + "\n") + assert "Traceback" not in result.output + assert result.exit_code == 0, result.output + + +def test_init_smoke_prints_skip_note(tmp_path, monkeypatch): + """A green smoke with skipped rows must say so (the doctor already + does): '(N check(s) skipped — not verifiable before run time)'.""" + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.chdir(tmp_path) + from benchflow.onboarding import CheckResult + + monkeypatch.setattr( + "benchflow.onboarding.run_doctor", + lambda *a, **k: [ + CheckResult("docker", True, "ok"), + CheckResult("provider auth", True, "skipped — sub", skipped=True), + ], + ) + result = runner.invoke(app, _init_args(tmp_path)[:-1]) # no --skip-smoke + assert result.exit_code == 0, result.output + assert "check(s) skipped" in result.output + + +def test_non_tty_env_source_announced_exactly_once(tmp_path, monkeypatch): + monkeypatch.setenv("BENCHFLOW_HOME", str(tmp_path)) + monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-exported-key") + result = runner.invoke(app, [*_init_args(tmp_path)[:-3], "--skip-smoke"]) + assert result.exit_code == 0, result.output + assert result.output.count("Using DEEPSEEK_API_KEY from your environment") == 1 diff --git a/tests/test_onboarding.py b/tests/test_onboarding.py new file mode 100644 index 00000000..01f31f1a --- /dev/null +++ b/tests/test_onboarding.py @@ -0,0 +1,780 @@ +"""bench init / bench doctor onboarding logic (benchflow.onboarding). + +Behavior under test, not implementation: credentials land in a private env +file that later runs can source; preferences round-trip; provider/agent +selection follows the wire-protocol rules the registry already declares; the +smoke ping tells the truth about a working vs broken key; the wizard's output +is a runnable `bench eval run` command. +""" + +from __future__ import annotations + +from benchflow import onboarding + + +class TestEnvFile: + def test_write_creates_private_file_and_read_back(self, tmp_path): + path = tmp_path / ".benchflow" / ".env" + onboarding.write_env_file(path, {"DEEPSEEK_API_KEY": "sk-test"}) + assert path.read_text() == 'DEEPSEEK_API_KEY="sk-test"\n' + assert (path.stat().st_mode & 0o777) == 0o600 + + def test_write_merges_and_overwrites_only_given_keys(self, tmp_path): + path = tmp_path / ".env" + onboarding.write_env_file(path, {"A": "1", "B": "2"}) + onboarding.write_env_file(path, {"B": "changed"}) + content = onboarding.read_env_file(path) + assert content == {"A": "1", "B": "changed"} + + def test_load_into_environ_never_overrides_real_env(self, tmp_path, monkeypatch): + path = tmp_path / ".env" + onboarding.write_env_file( + path, {"BF_PROBE_FROM_FILE": "file", "BF_PROBE_SET": "file"} + ) + monkeypatch.delenv("BF_PROBE_FROM_FILE", raising=False) + monkeypatch.setenv("BF_PROBE_SET", "real-env") + loaded = onboarding.load_env_file(path) + import os + + assert os.environ["BF_PROBE_FROM_FILE"] == "file" + assert os.environ["BF_PROBE_SET"] == "real-env" # file must not clobber + assert loaded == ["BF_PROBE_FROM_FILE"] + monkeypatch.delenv("BF_PROBE_FROM_FILE") + + +class TestPrefs: + def test_round_trip_and_missing_file_is_empty(self, tmp_path): + path = tmp_path / "config.toml" + assert onboarding.load_prefs(path) == {} + prefs = { + "agent": "pi-acp", + "model": "deepseek/deepseek-v4-flash", + "dataset": "skillsbench", + "sandbox": "docker", + } + onboarding.save_prefs(path, prefs) + assert onboarding.load_prefs(path) == prefs + + +class TestProviderResolution: + def test_prefixed_model_resolves_to_its_provider(self): + name, cfg = onboarding.resolve_provider("deepseek/deepseek-v4-flash") + assert name == "deepseek" + assert cfg.auth_env == "DEEPSEEK_API_KEY" + + def test_bare_model_resolves_via_model_prefixes(self): + name, _cfg = onboarding.resolve_provider("deepseek-v4-flash") + assert name == "deepseek" + + def test_unknown_model_returns_none(self): + assert onboarding.resolve_provider("definitely-not-a-model-9000") is None + + +class TestAgentPicker: + def test_deepseek_filter_excludes_wire_incompatible_agents(self): + """deepseek is openai-completions only: anthropic-messages and + openai-responses agents (claude-agent-acp, codex-acp) must not be + offered; chat-compatible core agents must be; oracle (no model) and + gemini (native provider protocol, bypasses routing) never appear.""" + names = onboarding.compatible_agents("deepseek/deepseek-v4-flash") + assert "pi-acp" in names and "opencode" in names + assert "claude-agent-acp" not in names + assert "codex-acp" not in names + assert "oracle" not in names and "gemini" not in names + + +class TestModelPing: + """A GET /models can 200 while the route is broken (proven in the census); + only a minimal completion exercises key + model id + endpoint.""" + + def _transport(self, status, body): + import httpx + + def handler(request): + # the ping must hit the chat-completions route with a minimal cap + assert request.url.path.endswith("/chat/completions") + import json + + payload = json.loads(request.content) + assert payload["max_tokens"] == 1 + return httpx.Response(status, json=body) + + return httpx.MockTransport(handler) + + def test_working_key_reports_ok(self): + result = onboarding.model_ping( + "deepseek/deepseek-v4-flash", + env={"DEEPSEEK_API_KEY": "sk-good"}, + transport=self._transport(200, {"choices": [{}], "model": "x"}), + ) + assert result.ok + assert "deepseek" in result.name + + def test_bad_key_reports_failure_with_status(self): + result = onboarding.model_ping( + "deepseek/deepseek-v4-flash", + env={"DEEPSEEK_API_KEY": "sk-bad"}, + transport=self._transport(401, {"error": {"message": "bad key"}}), + ) + assert not result.ok + assert "401" in result.detail + + def test_missing_key_fails_before_any_request(self): + result = onboarding.model_ping("deepseek/deepseek-v4-flash", env={}) + assert not result.ok + assert "DEEPSEEK_API_KEY" in result.detail + + +class TestDoctor: + def test_daytona_without_key_fails_and_docker_binary_checked(self, monkeypatch): + import httpx + + ok_transport = httpx.MockTransport( + lambda req: httpx.Response(200, json={"choices": [{}]}) + ) + monkeypatch.setattr("shutil.which", lambda _: None) + results = onboarding.run_doctor( + model="deepseek/deepseek-v4-flash", + sandbox="daytona", + env={"DEEPSEEK_API_KEY": "sk"}, + ping_transport=ok_transport, + ) + by_name = {r.name: r for r in results} + assert not by_name["daytona (DAYTONA_API_KEY)"].ok + assert "docker" not in " ".join(by_name) # daytona run: no docker row + assert by_name["provider key (DEEPSEEK_API_KEY)"].ok + assert by_name["model ping (deepseek)"].ok + + def test_docker_sandbox_checks_docker_binary(self, monkeypatch): + monkeypatch.setattr("shutil.which", lambda _: None) + results = onboarding.run_doctor( + model="deepseek/deepseek-v4-flash", + sandbox="docker", + env={}, + skip_ping=True, + ) + by_name = {r.name: r for r in results} + assert not by_name["docker"].ok + assert not by_name["provider key (DEEPSEEK_API_KEY)"].ok + assert "model ping (deepseek)" not in by_name + + +class TestCommandAssembly: + PREFS: dict = { # noqa: RUF012 — read-only test fixture + "agent": "pi-acp", + "model": "deepseek/deepseek-v4-flash", + "dataset": "skillsbench", + "sandbox": "docker", + "skill_mode": "with-skill", + } + + def test_final_command_is_a_runnable_eval_run(self): + cmd = onboarding.final_command(self.PREFS) + assert cmd == ( + "bench eval run --agent pi-acp --model deepseek/deepseek-v4-flash" + " -d skillsbench --sandbox docker --skill-mode with-skill" + ) + + def test_tasks_dir_dataset_uses_tasks_dir_flag(self): + prefs = {**self.PREFS, "dataset": "/data/my-tasks"} + assert " --tasks-dir /data/my-tasks " in onboarding.final_command(prefs) + " " + + def test_final_command_shell_quotes_task_paths_with_spaces(self): + """Guards PR #883: printed eval commands are copy-pasteable.""" + import shlex + + prefs = {**self.PREFS, "dataset": "/data/my tasks"} + cmd = onboarding.final_command(prefs) + parts = shlex.split(cmd) + assert "--tasks-dir '/data/my tasks'" in cmd + assert parts[parts.index("--tasks-dir") + 1] == "/data/my tasks" + + def test_oracle_smoke_argv_swaps_agent_and_pins_one_task(self): + argv = onboarding.smoke_argv(self.PREFS, task="citation-check") + assert argv[:4] == ["bench", "eval", "run", "--agent"] + assert argv[4] == "oracle" + assert "--include" in argv and "citation-check" in argv + assert "--model" not in argv # oracle needs no model + + +class TestEnvFileRobustness: + """Hand-edited dotenv dialects and corrupt files must never break the CLI + (the autoload callback runs on EVERY subcommand — including the ones + needed to fix the file).""" + + def test_export_prefix_and_single_quotes_parse(self, tmp_path): + path = tmp_path / ".env" + path.write_text("export DEEPSEEK_API_KEY=sk-abc\nZAI_API_KEY='sk-z'\n") + assert onboarding.read_env_file(path) == { + "DEEPSEEK_API_KEY": "sk-abc", + "ZAI_API_KEY": "sk-z", + } + + def test_malformed_lines_are_skipped_not_fatal(self, tmp_path, monkeypatch): + path = tmp_path / ".env" + path.write_text('=oops\n =also bad\nGOOD_KEY="v"\nBAD KEY=x\n') + monkeypatch.delenv("GOOD_KEY", raising=False) + applied = onboarding.load_env_file(path) # must not raise OSError + assert applied == ["GOOD_KEY"] + monkeypatch.delenv("GOOD_KEY") + + def test_unreadable_file_warns_not_crashes(self, tmp_path): + path = tmp_path / ".env" + path.write_text("A=1") + path.chmod(0o000) + try: + assert onboarding.load_env_file(path) == [] # degraded, no raise + finally: + path.chmod(0o600) + + +class TestModelPingProviderClasses: + """The ping must use the SAME endpoint join as the run path — no URL + guessing — and be honest for provider classes it cannot exercise.""" + + def _capture(self, status=200, body=None): + import httpx + + seen = {} + + def handler(request): + import json + + seen["url"] = str(request.url) + seen["headers"] = dict(request.headers) + seen["json"] = json.loads(request.content) + return httpx.Response(status, json=body or {"choices": [{}]}) + + return httpx.MockTransport(handler), seen + + def test_zai_versioned_base_gets_no_extra_v1(self): + transport, seen = self._capture() + result = onboarding.model_ping( + "zai/glm-5", env={"ZAI_API_KEY": "sk-z"}, transport=transport + ) + assert result.ok, result.detail + assert seen["url"] == "https://api.z.ai/api/paas/v4/chat/completions" + + def test_anthropic_only_provider_pings_messages_with_x_api_key(self): + transport, seen = self._capture(body={"content": [], "type": "message"}) + result = onboarding.model_ping( + "azure-foundry-anthropic/claude-opus-4-6", + env={"AZURE_API_KEY": "sk-az", "AZURE_RESOURCE": "myres"}, + transport=transport, + ) + assert result.ok, result.detail + assert seen["url"] == ( + "https://myres.services.ai.azure.com/anthropic/v1/messages" + ) + assert seen["headers"].get("x-api-key") == "sk-az" + + def test_azure_openai_ping_uses_api_key_header_and_gpt5_token_field(self): + """Guards PR #883: Azure GPT-5 smoke pings use supported wire shape.""" + transport, seen = self._capture() + result = onboarding.model_ping( + "azure-foundry-openai/gpt-5.5", + env={"AZURE_API_KEY": "sk-az", "AZURE_RESOURCE": "myres"}, + transport=transport, + ) + assert result.ok, result.detail + assert seen["url"] == ( + "https://myres.openai.azure.com/openai/v1/chat/completions" + ) + assert seen["headers"].get("api-key") == "sk-az" + assert "authorization" not in seen["headers"] + assert seen["json"]["max_completion_tokens"] == 8 + assert "max_tokens" not in seen["json"] + + def test_adc_provider_is_honestly_skipped_not_failed(self): + result = onboarding.model_ping("google-vertex/gemini-3-pro", env={}) + assert result.ok + assert "skipped" in result.detail + + def test_200_with_non_completion_body_fails(self): + import httpx + + transport = httpx.MockTransport( + lambda req: httpx.Response(200, text="login page") + ) + result = onboarding.model_ping( + "deepseek/deepseek-v4-flash", + env={"DEEPSEEK_API_KEY": "sk"}, + transport=transport, + ) + assert not result.ok + assert "not a completion" in result.detail + + def test_error_detail_strips_terminal_escapes(self): + import httpx + + transport = httpx.MockTransport( + lambda req: httpx.Response(500, text="bad \x1b]0;pwned\x07 thing") + ) + result = onboarding.model_ping( + "deepseek/deepseek-v4-flash", + env={"DEEPSEEK_API_KEY": "sk"}, + transport=transport, + ) + assert not result.ok + assert "\x1b" not in result.detail and "\x07" not in result.detail + + +class TestDoctorHardening: + def test_doctor_reuses_run_path_azure_endpoint_normalization(self, monkeypatch): + """Guards PR #883: saved Azure endpoint-only setup reaches smoke ping.""" + import httpx + + monkeypatch.delenv("AZURE_RESOURCE", raising=False) + monkeypatch.setattr("benchflow.agents.env.load_dotenv_env", lambda: {}) + seen = {} + + def handler(request): + import json + + seen["url"] = str(request.url) + seen["json"] = json.loads(request.content) + return httpx.Response(200, json={"choices": [{}]}) + + results = onboarding.run_doctor( + model="azure-foundry-openai/gpt-5.5", + sandbox="modal", + env={ + "AZURE_API_KEY": "sk-az", + "AZURE_API_ENDPOINT": "https://example-resource.openai.azure.com/", + }, + ping_transport=httpx.MockTransport(handler), + agent="pi-acp", + ) + + assert all(r.ok for r in results), [r for r in results if not r.ok] + assert seen["url"] == ( + "https://example-resource.openai.azure.com/openai/v1/chat/completions" + ) + assert seen["json"]["max_completion_tokens"] == 8 + + def test_unknown_sandbox_is_a_failing_row_not_silence(self): + results = onboarding.run_doctor( + model="deepseek/deepseek-v4-flash", + sandbox="dokcer", # typo + env={"DEEPSEEK_API_KEY": "sk"}, + skip_ping=True, + ) + row = next(r for r in results if r.name == "sandbox") + assert not row.ok + assert "dokcer" in row.detail + + def test_litellm_route_row_reports_resolution(self): + results = onboarding.run_doctor( + model="deepseek/deepseek-v4-flash", + sandbox="docker", + env={"DEEPSEEK_API_KEY": "sk"}, + skip_ping=True, + ) + row = next(r for r in results if r.name.startswith("litellm route")) + assert row.ok # deepseek resolves without network + + def test_unregistered_model_with_inferable_key_checks_that_key(self): + results = onboarding.run_doctor( + model="claude-opus-4-6", + sandbox="docker", + env={}, + skip_ping=True, + ) + row = next(r for r in results if "ANTHROPIC_API_KEY" in r.name) + assert not row.ok # key absent -> honest red row, not "no provider" + + +class TestEnvFileDataSafety: + def test_undecodable_file_degrades_never_crashes(self, tmp_path): + """One stray latin-1 byte must not brick every CLI command (the + autoload callback runs on ALL subcommands, including the repair + paths).""" + path = tmp_path / ".env" + path.write_bytes(b"NOTE=caf\xe9\nGOOD=1\n") + assert onboarding.load_env_file(path) == [] # degraded, no raise + + def test_rewrite_preserves_comments_and_unparsed_lines(self, tmp_path): + """write_env_file must not destroy what the parser skips — a hand + commented file survives the next `bench init` verbatim.""" + path = tmp_path / ".env" + path.write_text("# my deepseek key\nA=1\nnot a kv line\n") + onboarding.write_env_file(path, {"B": "2", "A": "changed"}) + text = path.read_text() + assert "# my deepseek key" in text + assert "not a kv line" in text + assert onboarding.read_env_file(path) == {"A": "changed", "B": "2"} + + def test_rewrite_refuses_to_clobber_undecodable_file(self, tmp_path): + path = tmp_path / ".env" + path.write_bytes(b"\xff\xfe binary junk") + import pytest + + with pytest.raises(OSError): + onboarding.write_env_file(path, {"A": "1"}) + assert path.read_bytes() == b"\xff\xfe binary junk" # untouched + + def test_rewrite_retightens_widened_permissions(self, tmp_path): + path = tmp_path / ".env" + onboarding.write_env_file(path, {"A": "1"}) + path.chmod(0o644) + onboarding.write_env_file(path, {"B": "2"}) + assert (path.stat().st_mode & 0o777) == 0o600 + + +class TestSubscriptionAwareDoctor: + def test_docker_row_uses_canonical_sandbox_preflight(self, monkeypatch): + """Guards PR #883: doctor must check Docker daemon readiness, not just + the docker binary path.""" + called = False + + def preflight(): + nonlocal called + called = True + + monkeypatch.setattr( + "benchflow.sandbox.docker.DockerSandbox.preflight", preflight + ) + results = onboarding.run_doctor( + model="deepseek/deepseek-v4-flash", + sandbox="docker", + env={"DEEPSEEK_API_KEY": "sk"}, + skip_ping=True, + ) + + assert called + row = next(r for r in results if r.name == "docker") + assert row.ok + assert "daemon ready" in row.detail + + def test_docker_row_reports_preflight_failure(self, monkeypatch): + """Guards PR #883 against a green doctor row when the Docker daemon is down.""" + + def preflight(): + raise SystemExit("Docker daemon is not running. Please start Docker.") + + monkeypatch.setattr( + "benchflow.sandbox.docker.DockerSandbox.preflight", preflight + ) + results = onboarding.run_doctor( + model="deepseek/deepseek-v4-flash", + sandbox="docker", + env={"DEEPSEEK_API_KEY": "sk"}, + skip_ping=True, + ) + + row = next(r for r in results if r.name == "docker") + assert not row.ok + assert "Docker daemon is not running" in row.detail + + def test_subscription_login_skips_key_route_and_ping_rows(self, monkeypatch): + """A subscription-onboarded setup (host login files, no API key) must + not be failed by its own smoke test: key/route/ping rows are skipped, + not red.""" + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.setattr( + "benchflow.agents.env.check_subscription_auth", lambda a, k: True + ) + results = onboarding.run_doctor( + model="claude-opus-4-6", + sandbox="docker", + env={}, + agent="claude-agent-acp", + ) + assert all(r.ok for r in results if r.name != "docker") + skipped = [r for r in results if r.skipped] + assert any("subscription" in r.detail for r in skipped) + + def test_skipped_rows_carry_the_skipped_flag(self): + result = onboarding.model_ping("google-vertex/gemini-3-pro", env={}) + assert result.ok and result.skipped + + def test_modal_sandbox_row_is_sane(self): + results = onboarding.run_doctor( + model="deepseek/deepseek-v4-flash", + sandbox="modal", + env={"DEEPSEEK_API_KEY": "sk"}, + skip_ping=True, + ) + row = next(r for r in results if r.name == "sandbox") + assert row.ok + assert "unknown" not in row.detail + + def test_route_row_failure_names_the_exception_type(self, monkeypatch): + def boom(model, env): + raise ValueError("boom") + + monkeypatch.setattr( + "benchflow.providers.litellm_config.resolve_litellm_route", boom + ) + results = onboarding.run_doctor( + model="deepseek/deepseek-v4-flash", + sandbox="docker", + env={"DEEPSEEK_API_KEY": "sk"}, + skip_ping=True, + ) + row = next(r for r in results if r.name.startswith("litellm route")) + assert not row.ok + assert "ValueError" in row.detail + + +class TestDetectKey: + """After the model is chosen the wizard must find credentials itself: + ./.env in the working folder, then the process environment (which includes + the saved ~/.benchflow/.env), then subscription login — prompting only when + all three miss.""" + + def test_exported_key_beats_subscription_matching_the_run_path( + self, monkeypatch, tmp_path + ): + """resolve_agent_env inherits an exported key into the agent env and + uses_native_subscription_auth then returns False — so at RUN time an + exported key wins over a subscription login. detect_key must report + the same order or init announces an auth source the run won't use.""" + monkeypatch.setattr( + "benchflow.agents.env.check_subscription_auth", lambda a, k: True + ) + monkeypatch.setenv("PROBE_KEY", "from-env") + source, value = onboarding.detect_key( + "PROBE_KEY", agent="claude-agent-acp", cwd=tmp_path + ) + assert source == "environment" and value == "from-env" + + def test_subscription_wins_when_no_key_is_set(self, monkeypatch, tmp_path): + monkeypatch.setattr( + "benchflow.agents.env.check_subscription_auth", lambda a, k: True + ) + monkeypatch.delenv("PROBE_KEY", raising=False) + source, value = onboarding.detect_key( + "PROBE_KEY", agent="claude-agent-acp", cwd=tmp_path + ) + assert source == "subscription" and value is None + + def test_cwd_dotenv_beats_process_env_matching_the_run_path( + self, monkeypatch, tmp_path + ): + """Guards PR #883: detect_key follows resolve_agent_env source order.""" + monkeypatch.setenv("PROBE_KEY", "from-env") + (tmp_path / ".env").write_text('PROBE_KEY="from-cwd"\n') + source, value = onboarding.detect_key("PROBE_KEY", cwd=tmp_path) + assert source == "./.env" and value == "from-cwd" + + def test_cwd_dotenv_passes_through(self, monkeypatch, tmp_path): + monkeypatch.delenv("PROBE_KEY", raising=False) + (tmp_path / ".env").write_text('PROBE_KEY="from-cwd"\n') + source, value = onboarding.detect_key("PROBE_KEY", cwd=tmp_path) + assert source == "./.env" and value == "from-cwd" + + def test_nothing_found(self, monkeypatch, tmp_path): + monkeypatch.delenv("PROBE_KEY", raising=False) + assert onboarding.detect_key("PROBE_KEY", cwd=tmp_path) == (None, None) + + +class TestDatasetChoices: + def test_newest_version_first_per_name(self, monkeypatch): + entries = [ + {"name": "skillsbench", "version": "1.0", "description": "old"}, + {"name": "skillsbench", "version": "1.1", "description": "new"}, + ] + monkeypatch.setattr( + "benchflow._utils.dataset_registry.load_registry", lambda src: entries + ) + choices = onboarding.dataset_choices() + assert choices[0][0] == "skillsbench@1.1" + assert "skillsbench@1.0" in [c[0] for c in choices] + + def test_registry_unreachable_degrades_to_empty(self, monkeypatch): + def boom(src): + raise OSError("offline") + + monkeypatch.setattr("benchflow._utils.dataset_registry.load_registry", boom) + assert onboarding.dataset_choices() == [] + + def test_malformed_registry_entries_degrade_not_crash(self, monkeypatch): + monkeypatch.setattr( + "benchflow._utils.dataset_registry.load_registry", + lambda src: ["not-a-dict", {"name": "ok", "version": "1.0"}], + ) + # must not raise; the well-formed entry may or may not survive + assert isinstance(onboarding.dataset_choices(), list) + + +class TestAcpAgents: + def test_lists_local_acp_agents_without_network(self, monkeypatch): + """acp_agents must NOT trigger the manifest autoload (no repo clone to + populate a menu) and must exclude the ai-sdk-*/omnigent-* paths.""" + from benchflow.agents import remote_manifests + + called = [] + monkeypatch.setattr( + remote_manifests, + "autoload_remote_manifest_agents", + lambda: called.append(1), + ) + names = onboarding.acp_agents() + assert called == [] # zero network + assert "pi-acp" in names + assert not any( + n == "ai-sdk" or n.startswith("ai-sdk-") or n.startswith("omnigent-") + for n in names + ) + assert "oracle" not in names and "gemini" not in names + + +class TestDetectKeySources: + def test_all_sources_listed_in_run_path_order(self, monkeypatch, tmp_path): + """The auth menu needs every detected source, ordered the way the run + path would use them: ./.env, environment, subscription. + + Guards PR #883 against validating one key while the final run uses + another. + """ + monkeypatch.setattr( + "benchflow.agents.env.check_subscription_auth", lambda a, k: True + ) + monkeypatch.setenv("PROBE_KEY", "from-env") + (tmp_path / ".env").write_text('PROBE_KEY="from-cwd"\n') + sources = onboarding.detect_key_sources( + "PROBE_KEY", agent="claude-agent-acp", cwd=tmp_path + ) + assert [s for s, _ in sources] == ["./.env", "environment", "subscription"] + + +class TestOfflineCatalogCache: + def test_source_root_falls_back_to_warm_cache_when_fetch_fails( + self, tmp_path, monkeypatch + ): + """A user who saw the full catalog online must not silently lose it + offline — the cached clone is the catalog of record.""" + from benchflow._utils import benchmark_repos + from benchflow.agents import remote_manifests + + cache = tmp_path / ".cache" / "datasets" / "benchflow-ai" / "agents" + cache.mkdir(parents=True) + monkeypatch.setattr( + benchmark_repos, "_cache_dir", lambda: tmp_path / ".cache" / "datasets" + ) + + def offline(repo, path=None, ref=None): + raise OSError("network down") + + monkeypatch.setattr(benchmark_repos, "resolve_source", offline) + root = remote_manifests._source_root("benchflow-ai/agents@main") + assert root == cache + + +class TestE2EFixCluster: + def test_cache_dir_never_falls_back_to_cwd(self, tmp_path, monkeypatch): + """Outside a git checkout the dataset/catalog cache must land in the + user cache dir, never pollute the working directory (e2e found a + 169MB clone dumped into a user's cwd).""" + from benchflow._utils import benchmark_repos + + monkeypatch.chdir(tmp_path) # no .git anywhere above tmp + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "xdg")) + cache = benchmark_repos._cache_dir() + assert not str(cache).startswith(str(tmp_path / ".cache")) + assert cache == tmp_path / "xdg" / "benchflow" / "datasets" + + def test_cache_dir_in_repo_checkout_unchanged(self, tmp_path, monkeypatch): + (tmp_path / ".git").mkdir() + monkeypatch.chdir(tmp_path) + from benchflow._utils import benchmark_repos + + assert benchmark_repos._cache_dir() == tmp_path / ".cache" / "datasets" + + def test_mimo_code_alias_is_builtin_and_local(self): + """mimo-code must resolve to the LOCAL canonical mimo without any + catalog fetch (the catalog's mimo-acp manifest claims the alias and + was silently winning).""" + from benchflow.agents.registry import AGENT_ALIASES + + assert AGENT_ALIASES.get("mimo-code") == "mimo" + + def test_dataset_descriptions_truncate_on_word_boundary(self, monkeypatch): + entries = [ + { + "name": "skillsbench", + "version": "1.1", + "description": "word " * 40, # 200 chars + } + ] + monkeypatch.setattr( + "benchflow._utils.dataset_registry.load_registry", lambda src: entries + ) + _spec, desc = onboarding.dataset_choices()[0] + assert len(desc) <= 80 + assert desc.endswith("…") + assert not desc.rstrip("…").endswith("wor") # no mid-word cut + + +class TestStaticCatalogLazyFetch: + """The wizard never loads the full agents repo: browsing is a STATIC + name list (zero network) and selecting fetches only that agent's + manifest.toml (single file), registered with local-wins semantics.""" + + def test_catalog_choices_is_static_and_networkless(self, monkeypatch): + from benchflow.agents import remote_manifests + + monkeypatch.setattr( + remote_manifests, + "autoload_remote_manifest_agents", + lambda: (_ for _ in ()).throw(AssertionError("network!")), + ) + from benchflow.agents.registry import AGENTS + + choices = onboarding.catalog_choices() + names = [n for n, _ in choices] + # exactly the static list minus whatever happens to be registered + # (suite order may have registered some catalog agents already) + assert names == [n for n in onboarding.CATALOG_AGENTS if n not in AGENTS] + assert len(onboarding.CATALOG_AGENTS) >= 20 + assert not any(n.startswith(("ai-sdk", "omnigent-")) for n in names) + + def test_fetch_one_registers_only_the_selected_agent(self, tmp_path, monkeypatch): + from benchflow.agents import registry, remote_manifests + + for name in ("probe-one", "probe-two"): + d = tmp_path / "acp" / name + d.mkdir(parents=True) + (d / "manifest.toml").write_text( + f'contract_version = "1.0"\nname = "{name}"\nprotocol = "acp"\n' + 'install_cmd = "true"\nlaunch_cmd = "true"\n' + ) + monkeypatch.setenv(remote_manifests.AGENTS_SOURCE_ENV, str(tmp_path)) + try: + assert remote_manifests.fetch_one("probe-one") is True + assert "probe-one" in registry.AGENTS + assert "probe-two" not in registry.AGENTS # ONLY the selected one + assert remote_manifests.fetch_one("no-such-agent") is False + finally: + registry.AGENTS.pop("probe-one", None) + registry.AGENT_INSTALLERS.pop("probe-one", None) + registry.AGENT_LAUNCH.pop("probe-one", None) + + def test_fetch_one_never_overwrites_local(self, tmp_path, monkeypatch): + from benchflow.agents import registry, remote_manifests + + d = tmp_path / "acp" / "mimo" + d.mkdir(parents=True) + (d / "manifest.toml").write_text( + 'contract_version = "1.0"\nname = "mimo"\nprotocol = "acp"\n' + 'install_cmd = "evil"\nlaunch_cmd = "evil"\n' + ) + monkeypatch.setenv(remote_manifests.AGENTS_SOURCE_ENV, str(tmp_path)) + before = registry.AGENTS["mimo"] + assert remote_manifests.fetch_one("mimo") is True # exists locally = fine + assert registry.AGENTS["mimo"] is before # untouched + + +class TestPingUnregisteredProvider: + def test_inferable_model_ping_is_skipped_not_failed(self): + """gemini-3-pro / claude-* run via inferred keys with no registered + endpooint to ping — the smoke must skip honestly, not fail a working + setup ('no registered provider' was a red row).""" + result = onboarding.model_ping("gemini-3-pro", env={"GEMINI_API_KEY": "k"}) + assert result.ok and result.skipped + assert "no registered endpoint" in result.detail + + def test_truly_unknown_model_still_fails(self): + result = onboarding.model_ping("no-such-model-9000", env={}) + assert not result.ok diff --git a/tests/test_task_download.py b/tests/test_task_download.py index a77bcb6f..4f4d5468 100644 --- a/tests/test_task_download.py +++ b/tests/test_task_download.py @@ -15,6 +15,15 @@ ) +@pytest.fixture(autouse=True) +def _cwd_cache(monkeypatch): + """These tests pin the clone cache to the (chdir'd) tmp cwd. Production + now routes to the user cache outside git checkouts — restore the + cwd-rooted behavior here since the subject under test is clone + mechanics, not cache placement policy.""" + monkeypatch.setattr(task_download, "_repo_root", lambda: Path.cwd()) + + def _fake_worktree(cmd): repo_root = Path(cmd[2]) snapshot = Path(cmd[-2]) @@ -47,6 +56,7 @@ def fake_run(cmd, check): [ "git", "clone", + "--quiet", "--depth", "1", "--branch", @@ -607,6 +617,7 @@ def fake_run(cmd, check): [ "git", "clone", + "--quiet", "--depth", "1", "https://github.com/org/repo.git",