Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
3c42271
feat(cli): bench init onboarding wizard + bench doctor
Jul 2, 2026
662679b
feat(cli): --full-smoke oracle stage for bench init
Jul 2, 2026
00143e3
fix(init): apply adversarially-verified review findings (16 confirmed)
Jul 2, 2026
a22aae5
fix(init): apply pr-review-toolkit findings (5 reviewers, 3 more real…
Jul 2, 2026
1fbfb59
feat(init): selection-driven wizard + automatic credential detection
Jul 3, 2026
4b22ddb
feat(init): agent-first wizard order
Jul 3, 2026
d76364c
fix(init): wizard review round — 4 adversarially-confirmed findings
Jul 3, 2026
1f538bf
feat(init): full 3-path catalog, path-first menus, auth-choice step, …
Jul 3, 2026
36144b0
fix(init): honor auth choices, autoload on the flags path, offline ca…
Jul 3, 2026
5489f75
fix(init): type-safe auth-menu branches + confirm line for the enviro…
Jul 3, 2026
2f3af72
fix(init): resolve PR 883 onboarding blockers
bingran-you Jul 3, 2026
da4df73
style(init): format PR 883 regression test
bingran-you Jul 3, 2026
fab137d
refactor(init): lazy agent catalog, ACP-only menu — no repo clone to …
Jul 3, 2026
ca6b16a
feat(init): native anthropic provider entry + catalog-browse for "other"
Jul 3, 2026
d7c950d
fix(init): apply the 13 confirmed findings from the all-agents e2e sweep
Jul 3, 2026
f08b4d6
style: RUF059 — unused unpacked var in dataset-ellipsis test
Jul 3, 2026
be96cb3
feat(init): static catalog + single-manifest lazy fetch; subscription
Jul 3, 2026
40dcd97
test: make the static-catalog test immune to suite-order registry pol…
Jul 3, 2026
acb98b0
style: B905 zip strict=True
Jul 3, 2026
179be2f
fix(init): line editing in prompts + 3-path static browse with per-path
Jul 5, 2026
dd506d5
fix(init): ping skips honestly for inferred-key model families
Jul 6, 2026
0622aec
fix(init): menu typo re-prompt, smoke skip-note, single source line, …
Jul 6, 2026
deb135d
Merge remote-tracking branch 'origin/main' into feat/bench-init
Jul 6, 2026
0ae90c6
merge main + fix flaky heartbeat test budget
Jul 6, 2026
c4a73d6
fix(onboarding): support GPT-5 ping shape
bingran-you Jul 7, 2026
a0e7c47
fix(onboarding): normalize azure setup env
bingran-you Jul 10, 2026
06071e7
test(onboarding): isolate subscription doctor env
bingran-you Jul 10, 2026
a75d65b
fix(onboarding): use docker preflight in doctor
bingran-you Jul 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 20 additions & 6 deletions src/benchflow/_utils/benchmark_repos.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,19 +47,30 @@ class ResolvedSource:
provenance: dict[str, Any]


def _repo_root() -> Path:
"""Find the repo root via .git directory."""
def _repo_root() -> Path | None:
"""Find the enclosing git checkout root, or None outside one."""
d = Path.cwd()
while d != d.parent:
if (d / ".git").exists():
return d
d = d.parent
return Path.cwd()
return None


def _cache_dir() -> Path:
"""Return the local cache directory for cloned dataset repos."""
return _repo_root() / ".cache" / "datasets"
"""Local cache directory for cloned dataset/catalog repos.

Inside a git checkout: ``<repo>/.cache/datasets`` (the dev workflow).
Outside one: the user cache (``$XDG_CACHE_HOME``/``~/.cache``) — NEVER
the working directory, which a pip-installed ``bench`` would otherwise
pollute with multi-hundred-MB clones.
"""
root = _repo_root()
if root is not None:
return root / ".cache" / "datasets"
xdg = os.environ.get("XDG_CACHE_HOME")
base = Path(xdg) if xdg else Path.home() / ".cache"
return base / "benchflow" / "datasets"


@contextmanager
Expand Down Expand Up @@ -155,7 +166,7 @@ def _clone_repo_unlocked(org: str, repo: str, ref: str | None = None) -> Path:
try:
if clone_tmp.exists():
shutil.rmtree(clone_tmp)
cmd = ["git", "clone", "--depth", "1"]
cmd = ["git", "clone", "--quiet", "--depth", "1"]
if ref and not _looks_like_commit_sha(ref):
cmd.extend(["--branch", ref])
cmd.extend([url, str(clone_tmp)])
Expand Down Expand Up @@ -474,6 +485,9 @@ def infer_task_source_provenance(task_path: Path) -> dict[str, Any] | None:
}

repo_root = _repo_root()
if repo_root is None:
# Not inside a git checkout: no repo provenance to derive.
return None
try:
rel = task_resolved.relative_to(repo_root.resolve(strict=False))
except ValueError:
Expand Down
3 changes: 3 additions & 0 deletions src/benchflow/agents/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -1026,6 +1026,9 @@ def infer_env_key_for_model(model: str) -> str | None:


AGENT_ALIASES: dict[str, str] = {
# mimo-code is the well-known vendor alias for the canonical mimo agent
# (the catalog's mimo-acp variant also claims it; local canonical wins).
"mimo-code": "mimo",
"claude": "claude-agent-acp",
"codex": "codex-acp",
"gemini": "gemini",
Expand Down
90 changes: 88 additions & 2 deletions src/benchflow/agents/remote_manifests.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,24 @@ def _source_root(spec: str) -> Path:
if local.is_dir():
return local
repo, _, ref = spec.partition("@")
from benchflow._utils.benchmark_repos import resolve_source
from benchflow._utils import benchmark_repos

return resolve_source(repo, ref=ref or None)
try:
return benchmark_repos.resolve_source(repo, ref=ref or None)
except Exception:
# Offline (or the refresh fetch failed): the cached clone is the
# catalog of record — a user who saw the full catalog online must
# not silently lose it, so fall back before giving up.
org, _, name = repo.partition("/")
cached = benchmark_repos._cache_dir() / org / name
if cached.is_dir():
logger.warning(
"could not refresh agents source %s; using the cached catalog at %s",
spec,
cached,
)
return cached
raise


def _gap_fill(
Expand Down Expand Up @@ -169,3 +184,74 @@ def _reset_for_tests() -> None:
global _attempted, last_source_description
_attempted = False
last_source_description = ""


def fetch_one(name: str) -> bool:
"""Fetch and register ONE catalog agent's manifest — never the full repo.

Local-dir sources read ``acp/<name>/manifest.toml`` directly; remote
``owner/repo[@ref]`` sources fetch that single file over HTTPS (raw
GitHub), so browsing/selecting an agent costs one small request instead
of a multi-hundred-MB clone. Local-wins semantics: a name already
registered is left untouched (returns True — the agent is available).
Returns False when the manifest cannot be fetched or is invalid.
"""
import os

from benchflow.agents import registry
from benchflow.agents.manifest import load_agent_manifest

if name in registry.AGENTS:
return True
spec = os.environ.get(AGENTS_SOURCE_ENV) or DEFAULT_AGENTS_SOURCE
if spec.strip().lower() in _OFF_VALUES:
return False
local = Path(spec).expanduser()
try:
if local.is_dir():
text = (local / "acp" / name / "manifest.toml").read_text()
else:
import httpx

repo, _, ref = spec.partition("@")
url = (
f"https://raw.githubusercontent.com/{repo}/{ref or 'main'}"
f"/acp/{name}/manifest.toml"
)
resp = httpx.get(url, timeout=30, follow_redirects=True)
if resp.status_code != 200:
logger.warning(
"catalog agent %r: manifest fetch returned HTTP %s (%s)",
name,
resp.status_code,
url,
)
return False
text = resp.text
except Exception as exc:
logger.warning("catalog agent %r: manifest fetch failed: %s", name, exc)
return False

import tempfile

with tempfile.TemporaryDirectory() as td:
mpath = Path(td) / "manifest.toml"
mpath.write_text(text)
try:
loaded = load_agent_manifest(mpath)
except Exception as exc:
logger.warning("catalog agent %r: invalid manifest: %s", name, exc)
return False
from benchflow.agents.manifest import register_manifest_agents

kept = _gap_fill([loaded], agents=registry.AGENTS, aliases=registry.AGENT_ALIASES)
if kept:
register_manifest_agents(
kept,
agents=registry.AGENTS,
aliases=registry.AGENT_ALIASES,
installers=registry.AGENT_INSTALLERS,
launch=registry.AGENT_LAUNCH,
)
logger.info("catalog agent %r: registered from single-manifest fetch", name)
return name in registry.AGENTS
Loading
Loading