fix(judge): add bounded correction retry for agent judge

CHANGELOG.md

@@ -11,6 +11,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `agent_judge` now uses stable criterion IDs and a strict JSON response
   contract, rejects malformed or semantically incomplete judge output, and
   always derives report labels and ordering from configured criteria.
+- `agent_judge` now makes one bounded output-correction attempt after strict
+  decoding or semantic validation fails. Both raw responses are preserved,
+  retry engine artifacts are isolated under `outputs/judge/run/retry/`, and
+  runtime-backed artifacts are snapshotted before a retry can overwrite them.
+  Independent correction runs begin at turn one, while aggregate Judge duration
+  and usage metrics cover the complete correction flow without changing report
+  schemas.
 - HTML and Markdown reports now identify the top-level duration as evaluation
   wall time and show per-case tested-agent execution time plus input, output,
   and total token usage. Benchmark cases include compact with-Skill,

e2e/agent_test.go

@@ -1581,6 +1581,14 @@ judge:
 	if strings.TrimSpace(grading.Expectations[0].Evidence) == "" {
 		t.Fatalf("expected non-empty judge evidence: %s", data)
 	}
+	rawResponsePath := filepath.Join(outputDir, "iteration-1", "qoder-contract", "with_skill", "outputs", "judge", "run", "raw-response-attempt-1.txt")
+	rawResponse, err := os.ReadFile(rawResponsePath)
+	if err != nil {
+		t.Fatalf("read Qoder Judge raw response artifact: %v", err)
+	}
+	if strings.TrimSpace(string(rawResponse)) == "" {
+		t.Fatalf("Qoder Judge raw response artifact is empty: %s", rawResponsePath)
+	}
 }
 
 // TestAgent_QoderCLI_NoneRuntime_FullRun tests qodercli agent with none runtime.

e2e/pipeline_test.go

@@ -104,6 +104,117 @@ func TestPipeline_FullRun_WithMockEngine(t *testing.T) {
 	}
 }
 
+// TestPipeline_AgentJudgeCorrectionRetry verifies the bounded correction path
+// through the real CLI pipeline with deterministic engine output.
+func TestPipeline_AgentJudgeCorrectionRetry(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	stateFile := filepath.Join(t.TempDir(), "judge-attempts.txt")
+	outputDir := t.TempDir()
+	writeFile(t, filepath.Join(dir, "SKILL.md"), "# Judge correction fixture\n")
+	writeFile(t, filepath.Join(dir, "evals", "eval.yaml"), `schema_version: v1alpha1
+
+environment:
+  type: none
+
+skills:
+  - source: local_path
+    path: .
+
+engine:
+  name: qoder-cli
+  model:
+    provider: qoder
+    name: auto
+
+cases:
+  files:
+    - evals/cases/correction.yaml
+  defaults:
+    timeout_seconds: 30
+    max_turns: 1
+  parallelism: 1
+
+report:
+  formats: [json]
+`)
+	criterion := "The final response contains the marker MOCK_CORRECTION_MARKER."
+	writeFile(t, filepath.Join(dir, "evals", "cases", "correction.yaml"), `id: judge-correction
+title: Agent judge correction retry
+input:
+  prompt: Return the marker MOCK_CORRECTION_MARKER.
+constraints:
+  timeout_seconds: 30
+  max_turns: 1
+judge:
+  type: agent_judge
+  model: auto
+  criteria:
+    - "The final response contains the marker MOCK_CORRECTION_MARKER."
+  pass_threshold: 1.0
+`)
+
+	result := Run(t, RunConfig{
+		Env: mockEngineEnv(t,
+			"MOCK_JUDGE_CORRECTION_RETRY=1",
+			"MOCK_JUDGE_STATE_FILE="+stateFile,
+		),
+		WorkDir: dir,
+		Timeout: 60 * time.Second,
+	}, "run", filepath.Join(dir, "evals", "eval.yaml"), "--output-dir", outputDir, "--verbose")
+	if result.ExitCode != 0 {
+		t.Fatalf("correction retry run failed: exit=%d\nstdout=%s\nstderr=%s", result.ExitCode, result.Stdout, result.Stderr)
+	}
+
+	state, err := os.ReadFile(stateFile)
+	if err != nil {
+		t.Fatalf("read Judge attempt state: %v", err)
+	}
+	if string(state) != "initial\ncorrection\n" {
+		t.Fatalf("expected exactly one correction retry, got attempts %q", state)
+	}
+
+	caseDir := filepath.Join(outputDir, "iteration-1", "judge-correction", "with_skill")
+	gradingData, err := os.ReadFile(filepath.Join(caseDir, "grading.json"))
+	if err != nil {
+		t.Fatalf("read grading.json: %v", err)
+	}
+	var grading struct {
+		Expectations []struct {
+			Text     string `json:"text"`
+			Passed   bool   `json:"passed"`
+			Evidence string `json:"evidence"`
+		} `json:"expectations"`
+		Summary struct {
+			PassRate float64 `json:"pass_rate"`
+		} `json:"summary"`
+	}
+	if err := json.Unmarshal(gradingData, &grading); err != nil {
+		t.Fatalf("parse grading.json: %v", err)
+	}
+	if grading.Summary.PassRate != 1 || len(grading.Expectations) != 1 || !grading.Expectations[0].Passed {
+		t.Fatalf("expected corrected Judge PASS: %s", gradingData)
+	}
+	if grading.Expectations[0].Text != criterion || strings.TrimSpace(grading.Expectations[0].Evidence) == "" {
+		t.Fatalf("unexpected corrected expectation: %#v", grading.Expectations)
+	}
+
+	judgeDir := filepath.Join(caseDir, "outputs", "judge", "run")
+	for _, relativePath := range []string{
+		"stdout.json",
+		"raw-response-attempt-1.txt",
+		filepath.Join("retry", "stdout.json"),
+		"raw-response-attempt-2.txt",
+	} {
+		path := filepath.Join(judgeDir, relativePath)
+		info, err := os.Stat(path)
+		if err != nil || info.Size() == 0 {
+			t.Fatalf("expected non-empty Judge artifact %s: info=%v err=%v", relativePath, info, err)
+		}
+	}
+}
+
 // TestPipeline_MustContainPass verifies that a case with matching must_contain
 // keywords passes the expect check when the mock engine returns the right output.
 func TestPipeline_MustContainPass(t *testing.T) {

e2e/testdata/mock-engine/engine.sh

@@ -18,6 +18,8 @@
 #   MOCK_CREATE_FILE - if set, create this file in the current working directory
 #   MOCK_FAIL_COUNT  - fail the first N invocations, then succeed (requires MOCK_STATE_FILE)
 #   MOCK_STATE_FILE  - path to a temp file used as invocation counter for MOCK_FAIL_COUNT
+#   MOCK_JUDGE_CORRECTION_RETRY - emit an invalid initial Judge response and a valid correction
+#   MOCK_JUDGE_STATE_FILE       - optional file recording each Judge protocol attempt
 #
 # Judge prompt auto-detection:
 #   When the prompt contains "Required Response Format" AND "## Criteria",
@@ -80,6 +82,23 @@ if [[ -n "${MOCK_CREATE_FILE:-}" ]]; then
   echo "Generated by mock engine" > "$MOCK_CREATE_FILE"
 fi
 
+# Handle an agent_judge correction prompt before the initial-prompt detector:
+# fallback retries contain the original prompt in their message history.
+if [[ "${MOCK_JUDGE_CORRECTION_RETRY:-}" == "1" ]] && echo "$PROMPT" | grep -q "Agent Judge Output Correction"; then
+  if [[ -n "${MOCK_JUDGE_STATE_FILE:-}" ]]; then
+    echo "correction" >> "$MOCK_JUDGE_STATE_FILE"
+  fi
+  JSON_RESULTS=$(printf '%s' "$PROMPT" | python3 -c '
+import json, re, sys
+prompt = sys.stdin.read()
+ids = list(dict.fromkeys(re.findall(r"criterion_id\"\s*:\s*\"(criterion-[0-9]+)\"", prompt)))
+items = [{"criterion_id": criterion_id, "passed": True, "evidence": ["Mock engine corrected the response contract"], "failures": []} for criterion_id in ids]
+print(json.dumps({"results": items}))
+')
+  echo "$JSON_RESULTS"
+  exit "${MOCK_EXIT_CODE:-0}"
+fi
+
 # Handle agent_judge evaluation prompts.
 # The judge prompt always contains "Required Response Format" and "## Criteria".
 # We parse the stable criterion IDs and return a valid JSON response so that
@@ -97,6 +116,18 @@ for line in sys.stdin:
     items.append({"criterion_id": criterion_id, "passed": True, "evidence": ["Mock engine: criterion satisfied based on agent output analysis"], "failures": []})
 print(json.dumps({"results": items}))
 ')
+  if [[ "${MOCK_JUDGE_CORRECTION_RETRY:-}" == "1" ]]; then
+    if [[ -n "${MOCK_JUDGE_STATE_FILE:-}" ]]; then
+      echo "initial" >> "$MOCK_JUDGE_STATE_FILE"
+    fi
+    JSON_RESULTS=$(printf '%s' "$JSON_RESULTS" | python3 -c '
+import json, sys
+payload = json.load(sys.stdin)
+for item in payload["results"]:
+    item.pop("passed", None)
+print(json.dumps(payload))
+')
+  fi
   echo "$JSON_RESULTS"
   exit "${MOCK_EXIT_CODE:-0}"
 fi

internal/agent/custom_test.go

@@ -32,6 +32,32 @@ func userMessages() []transcript.Message {
 	return []transcript.Message{{Role: transcript.RoleUser, Content: "review the diff"}}
 }
 
+func TestCustomAgent_RunLocal_PromptPreservesSerializedCorrection(t *testing.T) {
+	t.Parallel()
+	rt := newCustomTestRuntime(t)
+	correctionEnvelope := "original judge request\nprevious invalid response\ncorrection instructions"
+	ag := customLocalAgent(&config.CustomEngineConfig{
+		Transport:      "local",
+		ResponseFormat: "text",
+		Local: &config.CustomLocalConfig{
+			Command: "printf",
+			Args:    []string{"%s", "${prompt}"},
+		},
+	})
+
+	result, err := ag.Run(context.Background(), rt, ExecOptions{}, []transcript.Message{{
+		Role:    transcript.RoleUser,
+		Content: correctionEnvelope,
+		Turn:    2,
+	}})
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if result.FinalMessage != correctionEnvelope {
+		t.Fatalf("custom ${prompt} lost correction context: got %q want %q", result.FinalMessage, correctionEnvelope)
+	}
+}
+
 func TestCustomAgent_RunLocal_StdoutSessionResult(t *testing.T) {
 	t.Parallel()
 	rt := newCustomTestRuntime(t)

internal/evaluator/README.md

@@ -158,6 +158,13 @@ Artifacts include:
 - `stdout.txt` — standard output
 - `*.jsonl` — trace files
 
+For `agent_judge`, artifacts live under `outputs/judge/run/`. The first engine
+artifact paths remain unchanged and `raw-response-attempt-1.txt` is always
+written. When strict output correction is needed, the directory also contains
+`raw-response-attempt-2.txt` and a `retry/` subdirectory for the second engine
+invocation. Evaluator preserves this layout on both successful corrections and
+final Judge errors.
+
 ---
 
 ## Orchestration Flow