feat(bench): JIT arm from real aj compile (not a hardcoded skill)

cmd/aj/bench_cmd.go

@@ -163,15 +163,21 @@ func printComparison(c bench.Comparison, compileCost int) {
 
 // runGenCompare sweeps the repeat counts, regenerating a FRESH fixture for each
 // arm (baseline mutates the tree in place, so the JIT arm must not inherit it)
-// and installing the workflow skill for the JIT arm. Reports break-even per count.
+// and giving the JIT arm the workflow skill. Reports break-even per count.
+//
+// The JIT arm's skill comes from the real `aj compile` when the fixture is a
+// CompilableFixture (seed logs -> compile -> generated skill), otherwise from a
+// hand-written SkilledFixture skill. Compile cost defaults to what aj compile
+// actually spent (0 on the deterministic path) unless --compile-cost overrides.
 func runGenCompare(runner bench.Runner, shape string, counts []int, workdir string) error {
 	fixture, ok := bench.FixtureByShape(shape)
 	if !ok {
 		return fmt.Errorf("unknown --gen shape %q (have: %v)", shape, bench.FixtureShapes())
 	}
+	compilable, canCompile := fixture.(bench.CompilableFixture)
 	skilled, canSkill := fixture.(bench.SkilledFixture)
-	if !canSkill {
-		return fmt.Errorf("fixture %q has no skill to install; --gen --compare needs one", shape)
+	if !canCompile && !canSkill {
+		return fmt.Errorf("fixture %q has no skill for the JIT arm; --gen --compare needs one", shape)
 	}
 	if len(counts) == 0 {
 		counts = []int{3}
@@ -186,12 +192,25 @@ func runGenCompare(runner bench.Runner, shape string, counts []int, workdir stri
 		fmt.Printf("[AJ] Generated fixtures under %s\n", base)
 	}
 
-	// The JIT arm installs the skill before its episode.
+	// realCompileCost is captured by the JIT setup when using aj compile.
+	realCompileCost := 0
+	ajBin, _ := os.Executable() // this binary is `aj`
+
+	// The JIT arm gets the workflow skill before its episode: real compile when
+	// available (and record its true token cost), else the hand-written skill.
 	runner.Setup = func(task bench.Task, arm bench.Arm) error {
-		if arm == bench.ArmJIT {
-			return skilled.InstallSkill(task)
+		if arm != bench.ArmJIT {
+			return nil
 		}
-		return nil
+		if canCompile {
+			cost, err := bench.CompileSkill(context.Background(), ajBin, compilable, task)
+			if err != nil {
+				return err
+			}
+			realCompileCost = cost
+			return nil
+		}
+		return skilled.InstallSkill(task)
 	}
 
 	comparisons := make([]bench.Comparison, 0, len(counts))
@@ -211,8 +230,14 @@ func runGenCompare(runner bench.Runner, shape string, counts []int, workdir stri
 		jit.Task = baseline.Task
 		c := bench.Compare(baseline, jit)
 		comparisons = append(comparisons, c)
+		// Break-even uses an explicit --compile-cost, else the cost aj compile
+		// actually spent (captured by the JIT setup; 0 on the deterministic path).
+		cost := benchCompileCost
+		if cost == 0 {
+			cost = realCompileCost
+		}
 		if !benchJSON {
-			printComparison(c, benchCompileCost)
+			printComparison(c, cost)
 		}
 	}
 	if benchJSON {

docs/superpowers/specs/2026-07-06-benchmark-findings.md

@@ -9,14 +9,17 @@
 
 ## TL;DR
 
-On the first repetition-parameterized workflow (`nullcheck`), a compiled skill produced a
-**~0.15% token saving** (354 tokens out of ~236k) at equal (100%) success. With any realistic
-compile cost, **break-even is thousands of invocations — i.e. not worth compiling for this shape.**
+On two trivial repetition workflows — `nullcheck` (hand-written skill) and `shellseq` (a **real
+`aj compile`** skill) — a skill produced a **~0% token saving** (354 and 19–35 tokens on ~236k / ~93k
+baselines) at equal (100%) success. With any realistic compile cost, **break-even is thousands of
+invocations — i.e. not worth compiling for these shapes.**
 
 This is a *useful* result, not a disappointing one: the benchmark exists to answer "does a skill
 actually save tokens, and after how many uses does it pay off?" — and here the honest answer is "no,
-not on a trivial edit." The methodology (measure at iso-accuracy from API-reported usage, never
-estimate) is what makes that answer trustworthy.
+not on a trivial edit or a short command sequence." It holds for a genuinely compiled skill, not just a
+hand-written stand-in. The methodology (measure at iso-accuracy from API-reported usage, never estimate)
+is what makes that answer trustworthy. The open question is whether an **exploration-heavy** workflow —
+where the baseline burns tokens *finding* what to change — flips the result.
 
 ## Method (as implemented)
 
@@ -66,21 +69,50 @@ almost no exploration, so there is nothing to save. A skill should show a real d
 the baseline burns tokens *finding* what to change — e.g. `migrate-N`: locate every call site of an API
 across N files, then apply a mechanical change. That is the natural next fixture.
 
+## Update — real `aj compile` skill (`shellseq`, 2026-07-07)
+
+The `nullcheck` result above used a *hand-written* skill. To test AgentJIT's own compiler, the `shellseq`
+fixture uses a repetitive shell workflow (`mkdir -p out` + N `touch`) that the **deterministic** compiler
+turns into a real skill: the JIT arm seeds session logs, runs the actual `aj compile`, and installs the
+generated `mkdir-out/scripts/mkdir-out.sh`. Compile cost is **0** (deterministic path is zero-token).
+
+Measured at **`--rollouts 3`** (mean ≈ median, so the single-rollout cache noise is gone):
+
+```
+shellseq n=2:  baseline 92,729 (med 92,704)  vs  jit 92,694 (med 92,694)  →  saving  35/use
+shellseq n=4:  baseline 92,808 (med 92,810)  vs  jit 92,789 (med 92,792)  →  saving  19/use
+```
+
+Both arms 100% at iso-accuracy. **The genuinely compiled skill saves ~0% (19–35 tokens on a ~92.7k
+baseline)** — even less than the hand-written `nullcheck` skill (354), and the saving *shrinks* as N
+grows. This confirms the conclusion holds for a **real** compiled skill, not just a stand-in: on a short
+known command sequence there is nothing to save, because baseline T2S is dominated by fixed context load.
+
+**Only Bash-shaped workflows are deterministically compilable** — the deterministic backend emits only
+Bash steps, so an Edit-based workflow like `nullcheck` routes to the LLM compiler instead (which is why
+`nullcheck` keeps a hand-written skill here).
+
+**Caveat on single-rollout runs:** baseline T2S is dominated by prompt-cache state that carries across
+the two sequential episodes, so `--rollouts 1` gives unreliable A/B numbers (a run once showed baseline
+0 vs jit 46k). Always use `--rollouts >= 3`; the harness reports mean/median/min/max.
+
 ## Reproducing
 
 ```
 # isolated sandbox; real data untouched
-AJ_HOME=$(mktemp -d) aj bench --gen nullcheck --n 1,2 --compare --rollouts 3
+AJ_HOME=$(mktemp -d) aj bench --gen nullcheck --n 1,2 --compare --rollouts 3   # hand-written skill
+AJ_HOME=$(mktemp -d) aj bench --gen shellseq  --n 2,4 --compare --rollouts 3   # real aj compile skill
 ```
 
 Flags: `--tasks | --gen`, `--arm baseline|jit`, `--compare`, `--rollouts`, `--n` (curve),
 `--compile-cost` (auto-read from `$AJ_HOME` stats if unset), `--dry-run`, `--json`.
 
 ## Limitations / next steps
 
-- Single trivial shape so far; add `migrate-N` (exploration-heavy) to find where JIT pays off.
-- Skill is a hand-written project-local SKILL.md, not one produced by `aj compile` from real logs;
-  a follow-up should close that loop so compile cost is measured from an actual compilation.
-- Rollouts here were 1 for cost/time; production numbers should average N≥3 and report the spread
-  (the harness already computes mean/median/min/max).
+- Two trivial shapes so far (`nullcheck`, `shellseq`), both ~0% saving. Add an **exploration-heavy**
+  shape (e.g. `migrate-N`: locate every call site across N files, then apply a mechanical change) — the
+  hypothesis is that JIT only pays off where the baseline burns tokens *finding* what to change.
+- **Addressed 2026-07-07:** the JIT skill can now come from real `aj compile` (`shellseq`), and results
+  are reported at `--rollouts 3`. Still open: only *deterministic* (Bash) workflows are compiled this way;
+  an LLM-compiled skill (for Edit-based workflows) would have a real, non-zero compile cost to measure.
 - Phase 4 (replay real `aj bootstrap` transcripts) remains, as an external-validity check.

internal/bench/compile_fixture.go

@@ -0,0 +1,121 @@
+package bench
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/agent-jit/agentjit/internal/ingest"
+)
+
+// CompilableFixture is a Fixture whose workflow is a repetitive shell sequence
+// that AgentJIT's deterministic compiler can turn into a real skill. Unlike
+// SkilledFixture (which installs a hand-written SKILL.md), the JIT arm here runs
+// the real `aj compile` over seeded session logs, so the benchmark measures
+// AgentJIT's own compiled skill — not a stand-in.
+type CompilableFixture interface {
+	Fixture
+	// SeedSessions writes session logs of the workflow under logsDir/<date>/,
+	// enough repetitions that the trace analyzer detects a hot path
+	// (MinPatternFrequency, default 3).
+	SeedSessions(logsDir string) error
+	// SkillName is the deterministic skill name aj compile will produce for this
+	// workflow (derived from the first Bash step), so the JIT arm can locate it.
+	SkillName() string
+}
+
+// ShellSeqFixture has a workflow that is a fixed sequence of shell commands
+// creating N marker files. AgentJIT compiles the repeated sequence into a
+// shell-script skill; the JIT arm can then invoke that script instead of issuing
+// each command.
+type ShellSeqFixture struct{}
+
+func (ShellSeqFixture) Shape() string { return "shellseq" }
+
+// SkillName matches inferSkillName's output for this workflow's first Bash step
+// ("mkdir -p out" -> tokens "mkdir","out" joined -> "mkdir-out").
+func (ShellSeqFixture) SkillName() string { return "mkdir-out" }
+
+// commands returns the workflow's shell steps for repeat count n: make an output
+// dir, then create n numbered marker files.
+func (ShellSeqFixture) commands(n int) []string {
+	cmds := []string{"mkdir -p out"}
+	for i := 0; i < n; i++ {
+		cmds = append(cmds, fmt.Sprintf("touch out/marker%d.txt", i))
+	}
+	return cmds
+}
+
+func (f ShellSeqFixture) SeedSessions(logsDir string) error {
+	// 4 sessions (> MinPatternFrequency=3) all repeating the same command shape,
+	// so the deterministic compiler detects and compiles the hot path.
+	base := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
+	for s := 0; s < 4; s++ {
+		sid := fmt.Sprintf("cld_shellseq%d", s)
+		dateDir := filepath.Join(logsDir, base.Format("2006-01-02"))
+		if err := os.MkdirAll(dateDir, 0o755); err != nil {
+			return err
+		}
+		var lines []string
+		for i, cmd := range f.commands(2) { // seed with a fixed shape (n=2)
+			ev := ingest.Event{
+				Timestamp:        base.Add(time.Duration(s) * time.Minute).Add(time.Duration(i) * time.Second),
+				SessionID:        sid,
+				Harness:          "claude-code",
+				EventType:        "post_tool_use",
+				ToolName:         "Bash",
+				ToolInput:        map[string]interface{}{"command": cmd},
+				WorkingDirectory: "/tmp/shellseq",
+			}
+			b, err := json.Marshal(ev)
+			if err != nil {
+				return err
+			}
+			lines = append(lines, string(b))
+		}
+		content := ""
+		for _, l := range lines {
+			content += l + "\n"
+		}
+		if err := os.WriteFile(filepath.Join(dateDir, sid+".jsonl"), []byte(content), 0o644); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (f ShellSeqFixture) Generate(dir string, n int) (Task, error) {
+	if n < 1 {
+		return Task{}, fmt.Errorf("shellseq fixture needs n >= 1, got %d", n)
+	}
+	target := filepath.Join(dir, "shellseq")
+	if err := os.MkdirAll(target, 0o755); err != nil {
+		return Task{}, err
+	}
+
+	// Verifier: exactly n marker files exist under out/.
+	verifyScript := fmt.Sprintf(
+		`test "$(ls out/marker*.txt 2>/dev/null | wc -l | tr -d '[:space:]')" = "%d"`, n)
+
+	cmds := f.commands(n)
+	return Task{
+		ID:      fmt.Sprintf("shellseq-%d", n),
+		Shape:   "shellseq",
+		RepoDir: target,
+		Prompt: fmt.Sprintf(
+			"In this directory, create an `out/` directory and %d empty marker files "+
+				"named out/marker0.txt .. out/marker%d.txt. The exact command sequence is:\n%s",
+			n, n-1, joinLines(cmds)),
+		Verify: Verification{Command: []string{"sh", "-c", verifyScript}},
+	}, nil
+}
+
+func joinLines(xs []string) string {
+	out := ""
+	for _, x := range xs {
+		out += "  " + x + "\n"
+	}
+	return out
+}

internal/bench/compile_runner.go

@@ -0,0 +1,88 @@
+package bench
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/agent-jit/agentjit/internal/config"
+)
+
+// CompileSkill runs the real `aj compile` over a CompilableFixture's seeded
+// session logs in an isolated AJ_HOME, then makes the generated skill available
+// to the task's episode (copied into task.RepoDir/.claude/skills). It returns
+// the compile cost in tokens (0 for the deterministic path) read from the
+// sandbox stats.
+//
+// ajBin is the path to the aj binary to invoke (so the benchmark exercises the
+// real compiler, not a reimplementation).
+func CompileSkill(ctx context.Context, ajBin string, fx CompilableFixture, task Task) (int, error) {
+	ajHome, err := os.MkdirTemp("", "aj-compile-")
+	if err != nil {
+		return 0, err
+	}
+	paths := config.PathsFromRoot(ajHome)
+	if err := paths.EnsureDirs(); err != nil {
+		return 0, err
+	}
+
+	// 1. Seed the workflow's session logs so there's a pattern to detect.
+	if err := fx.SeedSessions(paths.Logs); err != nil {
+		return 0, fmt.Errorf("seeding sessions: %w", err)
+	}
+
+	// 2. Run the real aj compile against that sandbox.
+	cmd := exec.CommandContext(ctx, ajBin, "compile")
+	cmd.Env = append(os.Environ(), "AJ_HOME="+ajHome)
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+	if err := cmd.Run(); err != nil {
+		return 0, fmt.Errorf("aj compile: %w\n%s", err, out.String())
+	}
+
+	// 3. Locate the generated skill and copy it into the task's project skills.
+	srcSkill := filepath.Join(paths.Skills, fx.SkillName())
+	if _, err := os.Stat(srcSkill); err != nil {
+		return 0, fmt.Errorf("aj compile produced no skill %q under %s:\n%s", fx.SkillName(), paths.Skills, out.String())
+	}
+	dstSkill := filepath.Join(task.RepoDir, ".claude", "skills", fx.SkillName())
+	if err := copyTree(srcSkill, dstSkill); err != nil {
+		return 0, fmt.Errorf("installing compiled skill: %w", err)
+	}
+
+	// 4. Read the real compile cost (0 for the zero-token deterministic path).
+	cost, err := CompileCostFromStats(paths.Stats)
+	if err != nil {
+		return 0, err
+	}
+	return cost, nil
+}
+
+// copyTree recursively copies a directory tree.
+func copyTree(src, dst string) error {
+	return filepath.Walk(src, func(p string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		rel, err := filepath.Rel(src, p)
+		if err != nil {
+			return err
+		}
+		target := filepath.Join(dst, rel)
+		if info.IsDir() {
+			return os.MkdirAll(target, 0o755)
+		}
+		data, err := os.ReadFile(p)
+		if err != nil {
+			return err
+		}
+		if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+			return err
+		}
+		return os.WriteFile(target, data, info.Mode())
+	})
+}