Skip to content

Commit 96b89b7

Browse files
minixalphaminixalpha
authored
docs(benchmarks): record the v0.9.0 pilot baseline (#44)
docs(benchmarks): record the v0.9.0 pilot baseline
2 parents c060536 + 256c342 commit 96b89b7

12 files changed

Lines changed: 3096 additions & 0 deletions

File tree

‎benchmarks/harbor/configs/v0.9.0-pilot20/README.md‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Frozen v0.9.0 pilot20 configuration
2+
3+
These files define one 20-task baseline for the published v0.9.0 agent, before
4+
0.10.0 development. `baseline.json` records the release revision, original task
5+
revisions, task image digests, source/test hashes, budgets, and failure policy.
6+
The four budget groups are disjoint and cover the same 20 tasks as install-only.
7+
8+
Copy this configuration directory into a checkout at the recorded v0.9.0
9+
revision, then run from that repository root using the standard Harbor adapter.
10+
The runner builds the agent from its current checkout. Configure an official
11+
DeepSeek API key in the environment and
12+
set `ANTHROPIC_BASE_URL=https://api.deepseek.com/anthropic`. Leave
13+
`ANTHROPIC_MODEL` unset so the explicit configuration selects `deepseek-flash`.
14+
See the [adapter documentation](../../README.md) for dependency and cache setup.
15+
16+
Run dependency preflight before any model work:
17+
18+
```bash
19+
uv run --project benchmarks/harbor python -m harbor_adapter run \
20+
--config benchmarks/harbor/configs/v0.9.0-pilot20/install-only.json \
21+
--install-only
22+
```
23+
24+
After preflight succeeds, run these four commands sequentially. The adapter
25+
generates a fresh job name when `--job-name` is omitted.
26+
27+
```bash
28+
uv run --project benchmarks/harbor python -m harbor_adapter run \
29+
--config benchmarks/harbor/configs/v0.9.0-pilot20/budget-720s.json
30+
uv run --project benchmarks/harbor python -m harbor_adapter run \
31+
--config benchmarks/harbor/configs/v0.9.0-pilot20/budget-1620s.json
32+
uv run --project benchmarks/harbor python -m harbor_adapter run \
33+
--config benchmarks/harbor/configs/v0.9.0-pilot20/budget-2220s.json
34+
uv run --project benchmarks/harbor python -m harbor_adapter run \
35+
--config benchmarks/harbor/configs/v0.9.0-pilot20/budget-3420s.json
36+
```
37+
38+
The agent setup timeout is 1,800 seconds. Each task retains its native work and
39+
per-attempt verifier limits; the agent work budget reserves 180 seconds from the
40+
native agent timeout. Concurrency is two, with one model run per task and no
41+
whole-trial retries. Only verifier timeouts may retry, at most three attempts on
42+
the same agent output with the original per-attempt limit.
43+
44+
At this revision, only the pinned QEMU task has a registered verifier dependency
45+
profile. The other 19 tasks must report `not_configured`, rather than being
46+
described as having passed verifier dependency preflight. Preserve every trial
47+
and report setup failures, verifier errors, and scored failures separately.

‎benchmarks/harbor/configs/v0.9.0-pilot20/baseline.json‎

Lines changed: 491 additions & 0 deletions
Large diffs are not rendered by default.

‎benchmarks/harbor/configs/v0.9.0-pilot20/budget-1620s.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"n_concurrent_trials": 2,
3+
"n_attempts": 1,
4+
"agents": [
5+
{
6+
"name": "harbor_adapter:NanoPyCodeAgent",
7+
"model_name": "deepseek/deepseek-flash",
8+
"override_setup_timeout_sec": 1800,
9+
"kwargs": {
10+
"max_turns": 100,
11+
"max_tokens": 65536,
12+
"time_budget_seconds": 1620
13+
}
14+
}
15+
],
16+
"tasks": [
17+
{
18+
"name": "terminal-bench/dna-assembly",
19+
"ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f"
20+
},
21+
{
22+
"name": "terminal-bench/path-tracing",
23+
"ref": "sha256:cf56094c881a488b27e9f204a638a7e78ed7d55e12dc3064108c93357190314c"
24+
},
25+
{
26+
"name": "terminal-bench/llm-inference-batching-scheduler",
27+
"ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67"
28+
}
29+
]
30+
}

‎benchmarks/harbor/configs/v0.9.0-pilot20/budget-2220s.json‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"n_concurrent_trials": 2,
3+
"n_attempts": 1,
4+
"agents": [
5+
{
6+
"name": "harbor_adapter:NanoPyCodeAgent",
7+
"model_name": "deepseek/deepseek-flash",
8+
"override_setup_timeout_sec": 1800,
9+
"kwargs": {
10+
"max_turns": 100,
11+
"max_tokens": 65536,
12+
"time_budget_seconds": 2220
13+
}
14+
}
15+
],
16+
"tasks": [
17+
{
18+
"name": "terminal-bench/schemelike-metacircular-eval",
19+
"ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6"
20+
}
21+
]
22+
}

‎benchmarks/harbor/configs/v0.9.0-pilot20/budget-3420s.json‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"n_concurrent_trials": 2,
3+
"n_attempts": 1,
4+
"agents": [
5+
{
6+
"name": "harbor_adapter:NanoPyCodeAgent",
7+
"model_name": "deepseek/deepseek-flash",
8+
"override_setup_timeout_sec": 1800,
9+
"kwargs": {
10+
"max_turns": 100,
11+
"max_tokens": 65536,
12+
"time_budget_seconds": 3420
13+
}
14+
}
15+
],
16+
"tasks": [
17+
{
18+
"name": "terminal-bench/regex-chess",
19+
"ref": "sha256:e763e0ac1c9759081af0a4a82ba51b8cf9ae5485a93de3bbe42d7d344597bd78"
20+
},
21+
{
22+
"name": "terminal-bench/caffe-cifar-10",
23+
"ref": "sha256:7b0045106d7d5af724efe96b610ba64f7893f5c88528401c573c4d47e384e2bf"
24+
},
25+
{
26+
"name": "terminal-bench/mteb-leaderboard",
27+
"ref": "sha256:484f6d7008a05b5b8640fc6618a384b8c9447cd76f85416c8a595028d29bff9c"
28+
},
29+
{
30+
"name": "terminal-bench/circuit-fibsqrt",
31+
"ref": "sha256:9bcffe1054bb33249aa578a9a2a74f3c8cca66b0cb7aa1328233f1d31822aae3"
32+
}
33+
]
34+
}

‎benchmarks/harbor/configs/v0.9.0-pilot20/budget-720s.json‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{
2+
"n_concurrent_trials": 2,
3+
"n_attempts": 1,
4+
"agents": [
5+
{
6+
"name": "harbor_adapter:NanoPyCodeAgent",
7+
"model_name": "deepseek/deepseek-flash",
8+
"override_setup_timeout_sec": 1800,
9+
"kwargs": {
10+
"max_turns": 100,
11+
"max_tokens": 65536,
12+
"time_budget_seconds": 720
13+
}
14+
}
15+
],
16+
"tasks": [
17+
{
18+
"name": "terminal-bench/write-compressor",
19+
"ref": "sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e"
20+
},
21+
{
22+
"name": "terminal-bench/torch-tensor-parallelism",
23+
"ref": "sha256:f32ce74a5aeb6638480247ab799fe46127bbee631acdd0921b0f394ec49b3684"
24+
},
25+
{
26+
"name": "terminal-bench/kv-store-grpc",
27+
"ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307"
28+
},
29+
{
30+
"name": "terminal-bench/pypi-server",
31+
"ref": "sha256:1a1e0542f58e2d3362fec17a9bbb98667717d9a4a3e9a4c8413d3150a4fa0ff1"
32+
},
33+
{
34+
"name": "terminal-bench/torch-pipeline-parallelism",
35+
"ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb"
36+
},
37+
{
38+
"name": "terminal-bench/qemu-alpine-ssh",
39+
"ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28"
40+
},
41+
{
42+
"name": "terminal-bench/openssl-selfsigned-cert",
43+
"ref": "sha256:d4afa2bd2a9ba1420db8d6cfde42ffdb4873ae2d955c35014e8da94444c83302"
44+
},
45+
{
46+
"name": "terminal-bench/log-summary-date-ranges",
47+
"ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6"
48+
},
49+
{
50+
"name": "terminal-bench/model-extraction-relu-logits",
51+
"ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17"
52+
},
53+
{
54+
"name": "terminal-bench/regex-log",
55+
"ref": "sha256:802c16cfd132e6c457529cb864be5a757c1b23b6cadc57f2d01983cb0110292a"
56+
},
57+
{
58+
"name": "terminal-bench/pytorch-model-recovery",
59+
"ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc"
60+
},
61+
{
62+
"name": "terminal-bench/merge-diff-arc-agi-task",
63+
"ref": "sha256:6aab6511a5344ce87698293bb1ce4cc51d9a45f1ad9f0c075d2a83197b36727d"
64+
}
65+
]
66+
}

‎benchmarks/harbor/configs/v0.9.0-pilot20/install-only.json‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
{
2+
"n_concurrent_trials": 2,
3+
"n_attempts": 1,
4+
"agents": [
5+
{
6+
"name": "harbor_adapter:NanoPyCodeAgent",
7+
"model_name": "deepseek/deepseek-flash",
8+
"override_setup_timeout_sec": 1800,
9+
"kwargs": {
10+
"max_turns": 100,
11+
"max_tokens": 65536,
12+
"time_budget_seconds": 720
13+
}
14+
}
15+
],
16+
"tasks": [
17+
{
18+
"name": "terminal-bench/write-compressor",
19+
"ref": "sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e"
20+
},
21+
{
22+
"name": "terminal-bench/torch-tensor-parallelism",
23+
"ref": "sha256:f32ce74a5aeb6638480247ab799fe46127bbee631acdd0921b0f394ec49b3684"
24+
},
25+
{
26+
"name": "terminal-bench/schemelike-metacircular-eval",
27+
"ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6"
28+
},
29+
{
30+
"name": "terminal-bench/kv-store-grpc",
31+
"ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307"
32+
},
33+
{
34+
"name": "terminal-bench/pypi-server",
35+
"ref": "sha256:1a1e0542f58e2d3362fec17a9bbb98667717d9a4a3e9a4c8413d3150a4fa0ff1"
36+
},
37+
{
38+
"name": "terminal-bench/dna-assembly",
39+
"ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f"
40+
},
41+
{
42+
"name": "terminal-bench/torch-pipeline-parallelism",
43+
"ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb"
44+
},
45+
{
46+
"name": "terminal-bench/qemu-alpine-ssh",
47+
"ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28"
48+
},
49+
{
50+
"name": "terminal-bench/openssl-selfsigned-cert",
51+
"ref": "sha256:d4afa2bd2a9ba1420db8d6cfde42ffdb4873ae2d955c35014e8da94444c83302"
52+
},
53+
{
54+
"name": "terminal-bench/regex-chess",
55+
"ref": "sha256:e763e0ac1c9759081af0a4a82ba51b8cf9ae5485a93de3bbe42d7d344597bd78"
56+
},
57+
{
58+
"name": "terminal-bench/log-summary-date-ranges",
59+
"ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6"
60+
},
61+
{
62+
"name": "terminal-bench/model-extraction-relu-logits",
63+
"ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17"
64+
},
65+
{
66+
"name": "terminal-bench/path-tracing",
67+
"ref": "sha256:cf56094c881a488b27e9f204a638a7e78ed7d55e12dc3064108c93357190314c"
68+
},
69+
{
70+
"name": "terminal-bench/regex-log",
71+
"ref": "sha256:802c16cfd132e6c457529cb864be5a757c1b23b6cadc57f2d01983cb0110292a"
72+
},
73+
{
74+
"name": "terminal-bench/caffe-cifar-10",
75+
"ref": "sha256:7b0045106d7d5af724efe96b610ba64f7893f5c88528401c573c4d47e384e2bf"
76+
},
77+
{
78+
"name": "terminal-bench/mteb-leaderboard",
79+
"ref": "sha256:484f6d7008a05b5b8640fc6618a384b8c9447cd76f85416c8a595028d29bff9c"
80+
},
81+
{
82+
"name": "terminal-bench/llm-inference-batching-scheduler",
83+
"ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67"
84+
},
85+
{
86+
"name": "terminal-bench/pytorch-model-recovery",
87+
"ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc"
88+
},
89+
{
90+
"name": "terminal-bench/circuit-fibsqrt",
91+
"ref": "sha256:9bcffe1054bb33249aa578a9a2a74f3c8cca66b0cb7aa1328233f1d31822aae3"
92+
},
93+
{
94+
"name": "terminal-bench/merge-diff-arc-agi-task",
95+
"ref": "sha256:6aab6511a5344ce87698293bb1ce4cc51d9a45f1ad9f0c075d2a83197b36727d"
96+
}
97+
],
98+
"install_only": true
99+
}

0 commit comments

Comments
 (0)