From 06aa61c2ea7dff3820b3e2f64ec9be138a9b22d5 Mon Sep 17 00:00:00 2001 From: Poytr1 Date: Tue, 7 Jul 2026 10:42:06 +0800 Subject: [PATCH 1/2] feat(bench): JIT arm from real `aj compile` (not a hardcoded skill) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers "is the benchmark's skill actually compiled by AgentJIT?" — now, yes, for compilable workflows. - CompilableFixture interface: SeedSessions (writes workflow session logs) + SkillName. ShellSeqFixture: a repetitive shell sequence (mkdir + N touches) that AgentJIT's deterministic compiler turns into a real shell-script skill. - CompileSkill: seeds logs into an isolated AJ_HOME, runs the real `aj compile`, copies the generated skill into the task's .claude/skills, and returns the true compile cost (0 on the deterministic path). - runGenCompare: for a CompilableFixture the JIT arm now runs real aj compile; break-even uses the actual compile cost unless --compile-cost overrides. NullCheckFixture (Edit-based) keeps its hand-written skill — the deterministic compiler only emits Bash steps, so it isn't compilable. Tests build the aj binary and run the real compile end-to-end: seed -> `aj compile` -> generated skill script installed, cost 0. Note on results: deterministic compile is zero-token, so break-even is immediate — but single-rollout A/B numbers are noisy because baseline T2S is dominated by prompt cache state that carries across sequential episodes. Use --rollouts >= 3 (harness already reports mean/median/spread). Refs: benchmark sandbox design; addresses the "hardcoded skill" limitation in docs/superpowers/specs/2026-07-06-benchmark-findings.md. Co-Authored-By: Claude Opus 4.8 --- cmd/aj/bench_cmd.go | 41 +++++++-- internal/bench/compile_fixture.go | 121 ++++++++++++++++++++++++++ internal/bench/compile_runner.go | 88 +++++++++++++++++++ internal/bench/compile_runner_test.go | 63 ++++++++++++++ internal/bench/fixture.go | 1 + 5 files changed, 306 insertions(+), 8 deletions(-) create mode 100644 internal/bench/compile_fixture.go create mode 100644 internal/bench/compile_runner.go create mode 100644 internal/bench/compile_runner_test.go diff --git a/cmd/aj/bench_cmd.go b/cmd/aj/bench_cmd.go index 68a2002..0ad366e 100644 --- a/cmd/aj/bench_cmd.go +++ b/cmd/aj/bench_cmd.go @@ -163,15 +163,21 @@ func printComparison(c bench.Comparison, compileCost int) { // runGenCompare sweeps the repeat counts, regenerating a FRESH fixture for each // arm (baseline mutates the tree in place, so the JIT arm must not inherit it) -// and installing the workflow skill for the JIT arm. Reports break-even per count. +// and giving the JIT arm the workflow skill. Reports break-even per count. +// +// The JIT arm's skill comes from the real `aj compile` when the fixture is a +// CompilableFixture (seed logs -> compile -> generated skill), otherwise from a +// hand-written SkilledFixture skill. Compile cost defaults to what aj compile +// actually spent (0 on the deterministic path) unless --compile-cost overrides. func runGenCompare(runner bench.Runner, shape string, counts []int, workdir string) error { fixture, ok := bench.FixtureByShape(shape) if !ok { return fmt.Errorf("unknown --gen shape %q (have: %v)", shape, bench.FixtureShapes()) } + compilable, canCompile := fixture.(bench.CompilableFixture) skilled, canSkill := fixture.(bench.SkilledFixture) - if !canSkill { - return fmt.Errorf("fixture %q has no skill to install; --gen --compare needs one", shape) + if !canCompile && !canSkill { + return fmt.Errorf("fixture %q has no skill for the JIT arm; --gen --compare needs one", shape) } if len(counts) == 0 { counts = []int{3} @@ -186,12 +192,25 @@ func runGenCompare(runner bench.Runner, shape string, counts []int, workdir stri fmt.Printf("[AJ] Generated fixtures under %s\n", base) } - // The JIT arm installs the skill before its episode. + // realCompileCost is captured by the JIT setup when using aj compile. + realCompileCost := 0 + ajBin, _ := os.Executable() // this binary is `aj` + + // The JIT arm gets the workflow skill before its episode: real compile when + // available (and record its true token cost), else the hand-written skill. runner.Setup = func(task bench.Task, arm bench.Arm) error { - if arm == bench.ArmJIT { - return skilled.InstallSkill(task) + if arm != bench.ArmJIT { + return nil } - return nil + if canCompile { + cost, err := bench.CompileSkill(context.Background(), ajBin, compilable, task) + if err != nil { + return err + } + realCompileCost = cost + return nil + } + return skilled.InstallSkill(task) } comparisons := make([]bench.Comparison, 0, len(counts)) @@ -211,8 +230,14 @@ func runGenCompare(runner bench.Runner, shape string, counts []int, workdir stri jit.Task = baseline.Task c := bench.Compare(baseline, jit) comparisons = append(comparisons, c) + // Break-even uses an explicit --compile-cost, else the cost aj compile + // actually spent (captured by the JIT setup; 0 on the deterministic path). + cost := benchCompileCost + if cost == 0 { + cost = realCompileCost + } if !benchJSON { - printComparison(c, benchCompileCost) + printComparison(c, cost) } } if benchJSON { diff --git a/internal/bench/compile_fixture.go b/internal/bench/compile_fixture.go new file mode 100644 index 0000000..e3310bb --- /dev/null +++ b/internal/bench/compile_fixture.go @@ -0,0 +1,121 @@ +package bench + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/agent-jit/agentjit/internal/ingest" +) + +// CompilableFixture is a Fixture whose workflow is a repetitive shell sequence +// that AgentJIT's deterministic compiler can turn into a real skill. Unlike +// SkilledFixture (which installs a hand-written SKILL.md), the JIT arm here runs +// the real `aj compile` over seeded session logs, so the benchmark measures +// AgentJIT's own compiled skill — not a stand-in. +type CompilableFixture interface { + Fixture + // SeedSessions writes session logs of the workflow under logsDir//, + // enough repetitions that the trace analyzer detects a hot path + // (MinPatternFrequency, default 3). + SeedSessions(logsDir string) error + // SkillName is the deterministic skill name aj compile will produce for this + // workflow (derived from the first Bash step), so the JIT arm can locate it. + SkillName() string +} + +// ShellSeqFixture has a workflow that is a fixed sequence of shell commands +// creating N marker files. AgentJIT compiles the repeated sequence into a +// shell-script skill; the JIT arm can then invoke that script instead of issuing +// each command. +type ShellSeqFixture struct{} + +func (ShellSeqFixture) Shape() string { return "shellseq" } + +// SkillName matches inferSkillName's output for this workflow's first Bash step +// ("mkdir -p out" -> tokens "mkdir","out" joined -> "mkdir-out"). +func (ShellSeqFixture) SkillName() string { return "mkdir-out" } + +// commands returns the workflow's shell steps for repeat count n: make an output +// dir, then create n numbered marker files. +func (ShellSeqFixture) commands(n int) []string { + cmds := []string{"mkdir -p out"} + for i := 0; i < n; i++ { + cmds = append(cmds, fmt.Sprintf("touch out/marker%d.txt", i)) + } + return cmds +} + +func (f ShellSeqFixture) SeedSessions(logsDir string) error { + // 4 sessions (> MinPatternFrequency=3) all repeating the same command shape, + // so the deterministic compiler detects and compiles the hot path. + base := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC) + for s := 0; s < 4; s++ { + sid := fmt.Sprintf("cld_shellseq%d", s) + dateDir := filepath.Join(logsDir, base.Format("2006-01-02")) + if err := os.MkdirAll(dateDir, 0o755); err != nil { + return err + } + var lines []string + for i, cmd := range f.commands(2) { // seed with a fixed shape (n=2) + ev := ingest.Event{ + Timestamp: base.Add(time.Duration(s) * time.Minute).Add(time.Duration(i) * time.Second), + SessionID: sid, + Harness: "claude-code", + EventType: "post_tool_use", + ToolName: "Bash", + ToolInput: map[string]interface{}{"command": cmd}, + WorkingDirectory: "/tmp/shellseq", + } + b, err := json.Marshal(ev) + if err != nil { + return err + } + lines = append(lines, string(b)) + } + content := "" + for _, l := range lines { + content += l + "\n" + } + if err := os.WriteFile(filepath.Join(dateDir, sid+".jsonl"), []byte(content), 0o644); err != nil { + return err + } + } + return nil +} + +func (f ShellSeqFixture) Generate(dir string, n int) (Task, error) { + if n < 1 { + return Task{}, fmt.Errorf("shellseq fixture needs n >= 1, got %d", n) + } + target := filepath.Join(dir, "shellseq") + if err := os.MkdirAll(target, 0o755); err != nil { + return Task{}, err + } + + // Verifier: exactly n marker files exist under out/. + verifyScript := fmt.Sprintf( + `test "$(ls out/marker*.txt 2>/dev/null | wc -l | tr -d '[:space:]')" = "%d"`, n) + + cmds := f.commands(n) + return Task{ + ID: fmt.Sprintf("shellseq-%d", n), + Shape: "shellseq", + RepoDir: target, + Prompt: fmt.Sprintf( + "In this directory, create an `out/` directory and %d empty marker files "+ + "named out/marker0.txt .. out/marker%d.txt. The exact command sequence is:\n%s", + n, n-1, joinLines(cmds)), + Verify: Verification{Command: []string{"sh", "-c", verifyScript}}, + }, nil +} + +func joinLines(xs []string) string { + out := "" + for _, x := range xs { + out += " " + x + "\n" + } + return out +} diff --git a/internal/bench/compile_runner.go b/internal/bench/compile_runner.go new file mode 100644 index 0000000..621e3f2 --- /dev/null +++ b/internal/bench/compile_runner.go @@ -0,0 +1,88 @@ +package bench + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/agent-jit/agentjit/internal/config" +) + +// CompileSkill runs the real `aj compile` over a CompilableFixture's seeded +// session logs in an isolated AJ_HOME, then makes the generated skill available +// to the task's episode (copied into task.RepoDir/.claude/skills). It returns +// the compile cost in tokens (0 for the deterministic path) read from the +// sandbox stats. +// +// ajBin is the path to the aj binary to invoke (so the benchmark exercises the +// real compiler, not a reimplementation). +func CompileSkill(ctx context.Context, ajBin string, fx CompilableFixture, task Task) (int, error) { + ajHome, err := os.MkdirTemp("", "aj-compile-") + if err != nil { + return 0, err + } + paths := config.PathsFromRoot(ajHome) + if err := paths.EnsureDirs(); err != nil { + return 0, err + } + + // 1. Seed the workflow's session logs so there's a pattern to detect. + if err := fx.SeedSessions(paths.Logs); err != nil { + return 0, fmt.Errorf("seeding sessions: %w", err) + } + + // 2. Run the real aj compile against that sandbox. + cmd := exec.CommandContext(ctx, ajBin, "compile") + cmd.Env = append(os.Environ(), "AJ_HOME="+ajHome) + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + return 0, fmt.Errorf("aj compile: %w\n%s", err, out.String()) + } + + // 3. Locate the generated skill and copy it into the task's project skills. + srcSkill := filepath.Join(paths.Skills, fx.SkillName()) + if _, err := os.Stat(srcSkill); err != nil { + return 0, fmt.Errorf("aj compile produced no skill %q under %s:\n%s", fx.SkillName(), paths.Skills, out.String()) + } + dstSkill := filepath.Join(task.RepoDir, ".claude", "skills", fx.SkillName()) + if err := copyTree(srcSkill, dstSkill); err != nil { + return 0, fmt.Errorf("installing compiled skill: %w", err) + } + + // 4. Read the real compile cost (0 for the zero-token deterministic path). + cost, err := CompileCostFromStats(paths.Stats) + if err != nil { + return 0, err + } + return cost, nil +} + +// copyTree recursively copies a directory tree. +func copyTree(src, dst string) error { + return filepath.Walk(src, func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, p) + if err != nil { + return err + } + target := filepath.Join(dst, rel) + if info.IsDir() { + return os.MkdirAll(target, 0o755) + } + data, err := os.ReadFile(p) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + return os.WriteFile(target, data, info.Mode()) + }) +} diff --git a/internal/bench/compile_runner_test.go b/internal/bench/compile_runner_test.go new file mode 100644 index 0000000..e7cb579 --- /dev/null +++ b/internal/bench/compile_runner_test.go @@ -0,0 +1,63 @@ +package bench + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" +) + +// buildAJ builds the aj binary into a temp dir and returns its path, or skips +// the test if the toolchain isn't available. +func buildAJ(t *testing.T) string { + t.Helper() + if _, err := exec.LookPath("go"); err != nil { + t.Skip("go toolchain not available") + } + dir := t.TempDir() + bin := filepath.Join(dir, "aj") + cmd := exec.Command("go", "build", "-o", bin, "github.com/agent-jit/agentjit/cmd/aj") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("building aj: %v\n%s", err, out) + } + return bin +} + +func TestShellSeqSeedSessions(t *testing.T) { + dir := t.TempDir() + if err := (ShellSeqFixture{}).SeedSessions(dir); err != nil { + t.Fatalf("SeedSessions: %v", err) + } + // 4 session files should exist under the date dir. + matches, _ := filepath.Glob(filepath.Join(dir, "*", "cld_shellseq*.jsonl")) + if len(matches) != 4 { + t.Errorf("seeded %d session files, want 4", len(matches)) + } +} + +// End-to-end: seed -> real `aj compile` (deterministic, zero tokens) -> the +// generated skill is installed into the task tree, cost is 0. +func TestCompileSkillProducesRealSkill(t *testing.T) { + ajBin := buildAJ(t) + fx := ShellSeqFixture{} + task, err := fx.Generate(t.TempDir(), 2) + if err != nil { + t.Fatal(err) + } + + cost, err := CompileSkill(context.Background(), ajBin, fx, task) + if err != nil { + t.Fatalf("CompileSkill: %v", err) + } + // Deterministic compile is zero-token. + if cost != 0 { + t.Errorf("compile cost = %d, want 0 (deterministic path)", cost) + } + // The compiled skill (with its generated shell script) landed in the task. + script := filepath.Join(task.RepoDir, ".claude", "skills", fx.SkillName(), "scripts", fx.SkillName()+".sh") + if _, err := os.Stat(script); err != nil { + t.Errorf("compiled skill script not installed at %s: %v", script, err) + } +} diff --git a/internal/bench/fixture.go b/internal/bench/fixture.go index 65b303c..78033b1 100644 --- a/internal/bench/fixture.go +++ b/internal/bench/fixture.go @@ -10,6 +10,7 @@ import ( // fixtures is the registry of built-in repetition fixtures, keyed by shape name. var fixtures = map[string]Fixture{ "nullcheck": NullCheckFixture{}, + "shellseq": ShellSeqFixture{}, } // FixtureByShape returns the built-in fixture for a shape name. From 4ba0b4f71afb85e739899269c41d6c7df7e42a6e Mon Sep 17 00:00:00 2001 From: Poytr1 Date: Tue, 7 Jul 2026 15:23:12 +0800 Subject: [PATCH 2/2] docs: add real-`aj compile` (shellseq) results to findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records the trustworthy --rollouts 3 numbers for the genuinely compiled skill: shellseq n=2 saves 35/use, n=4 saves 19/use (~0% on a ~93k baseline), both at iso-accuracy. Confirms the "trivial workflow → skill doesn't pay off" conclusion holds for a real compiled skill, not just the hand-written one. Marks the hardcoded-skill and rollouts=1 limitations as addressed; the exploration-heavy shape remains the open question. Co-Authored-By: Claude Opus 4.8 --- .../specs/2026-07-06-benchmark-findings.md | 54 +++++++++++++++---- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/docs/superpowers/specs/2026-07-06-benchmark-findings.md b/docs/superpowers/specs/2026-07-06-benchmark-findings.md index 78189cf..ae3d8aa 100644 --- a/docs/superpowers/specs/2026-07-06-benchmark-findings.md +++ b/docs/superpowers/specs/2026-07-06-benchmark-findings.md @@ -9,14 +9,17 @@ ## TL;DR -On the first repetition-parameterized workflow (`nullcheck`), a compiled skill produced a -**~0.15% token saving** (354 tokens out of ~236k) at equal (100%) success. With any realistic -compile cost, **break-even is thousands of invocations — i.e. not worth compiling for this shape.** +On two trivial repetition workflows — `nullcheck` (hand-written skill) and `shellseq` (a **real +`aj compile`** skill) — a skill produced a **~0% token saving** (354 and 19–35 tokens on ~236k / ~93k +baselines) at equal (100%) success. With any realistic compile cost, **break-even is thousands of +invocations — i.e. not worth compiling for these shapes.** This is a *useful* result, not a disappointing one: the benchmark exists to answer "does a skill actually save tokens, and after how many uses does it pay off?" — and here the honest answer is "no, -not on a trivial edit." The methodology (measure at iso-accuracy from API-reported usage, never -estimate) is what makes that answer trustworthy. +not on a trivial edit or a short command sequence." It holds for a genuinely compiled skill, not just a +hand-written stand-in. The methodology (measure at iso-accuracy from API-reported usage, never estimate) +is what makes that answer trustworthy. The open question is whether an **exploration-heavy** workflow — +where the baseline burns tokens *finding* what to change — flips the result. ## Method (as implemented) @@ -66,11 +69,39 @@ almost no exploration, so there is nothing to save. A skill should show a real d the baseline burns tokens *finding* what to change — e.g. `migrate-N`: locate every call site of an API across N files, then apply a mechanical change. That is the natural next fixture. +## Update — real `aj compile` skill (`shellseq`, 2026-07-07) + +The `nullcheck` result above used a *hand-written* skill. To test AgentJIT's own compiler, the `shellseq` +fixture uses a repetitive shell workflow (`mkdir -p out` + N `touch`) that the **deterministic** compiler +turns into a real skill: the JIT arm seeds session logs, runs the actual `aj compile`, and installs the +generated `mkdir-out/scripts/mkdir-out.sh`. Compile cost is **0** (deterministic path is zero-token). + +Measured at **`--rollouts 3`** (mean ≈ median, so the single-rollout cache noise is gone): + +``` +shellseq n=2: baseline 92,729 (med 92,704) vs jit 92,694 (med 92,694) → saving 35/use +shellseq n=4: baseline 92,808 (med 92,810) vs jit 92,789 (med 92,792) → saving 19/use +``` + +Both arms 100% at iso-accuracy. **The genuinely compiled skill saves ~0% (19–35 tokens on a ~92.7k +baseline)** — even less than the hand-written `nullcheck` skill (354), and the saving *shrinks* as N +grows. This confirms the conclusion holds for a **real** compiled skill, not just a stand-in: on a short +known command sequence there is nothing to save, because baseline T2S is dominated by fixed context load. + +**Only Bash-shaped workflows are deterministically compilable** — the deterministic backend emits only +Bash steps, so an Edit-based workflow like `nullcheck` routes to the LLM compiler instead (which is why +`nullcheck` keeps a hand-written skill here). + +**Caveat on single-rollout runs:** baseline T2S is dominated by prompt-cache state that carries across +the two sequential episodes, so `--rollouts 1` gives unreliable A/B numbers (a run once showed baseline +0 vs jit 46k). Always use `--rollouts >= 3`; the harness reports mean/median/min/max. + ## Reproducing ``` # isolated sandbox; real data untouched -AJ_HOME=$(mktemp -d) aj bench --gen nullcheck --n 1,2 --compare --rollouts 3 +AJ_HOME=$(mktemp -d) aj bench --gen nullcheck --n 1,2 --compare --rollouts 3 # hand-written skill +AJ_HOME=$(mktemp -d) aj bench --gen shellseq --n 2,4 --compare --rollouts 3 # real aj compile skill ``` Flags: `--tasks | --gen`, `--arm baseline|jit`, `--compare`, `--rollouts`, `--n` (curve), @@ -78,9 +109,10 @@ Flags: `--tasks | --gen`, `--arm baseline|jit`, `--compare`, `--rollouts`, `--n` ## Limitations / next steps -- Single trivial shape so far; add `migrate-N` (exploration-heavy) to find where JIT pays off. -- Skill is a hand-written project-local SKILL.md, not one produced by `aj compile` from real logs; - a follow-up should close that loop so compile cost is measured from an actual compilation. -- Rollouts here were 1 for cost/time; production numbers should average N≥3 and report the spread - (the harness already computes mean/median/min/max). +- Two trivial shapes so far (`nullcheck`, `shellseq`), both ~0% saving. Add an **exploration-heavy** + shape (e.g. `migrate-N`: locate every call site across N files, then apply a mechanical change) — the + hypothesis is that JIT only pays off where the baseline burns tokens *finding* what to change. +- **Addressed 2026-07-07:** the JIT skill can now come from real `aj compile` (`shellseq`), and results + are reported at `--rollouts 3`. Still open: only *deterministic* (Bash) workflows are compiled this way; + an LLM-compiled skill (for Edit-based workflows) would have a real, non-zero compile cost to measure. - Phase 4 (replay real `aj bootstrap` transcripts) remains, as an external-validity check.