diff --git a/docs/guide/writing-evals.md b/docs/guide/writing-evals.md index fc2e508f..af44b550 100644 --- a/docs/guide/writing-evals.md +++ b/docs/guide/writing-evals.md @@ -712,6 +712,10 @@ judge: all: ["bug", "null"] # Must contain ALL any: ["suggest fix", "recommend"] # Must contain at least one not: ["LGTM"] # Must NOT contain + - output_matches: # Go regexp patterns against final output + all: ["(?m)^## Status$", "(?m)^## Evidence$"] + any: ["(?i)pass", "(?i)success"] + not: ["(?i)api[_-]?key\\s*="] - exit_code: 0 - tool_called: # Agent must invoke this tool name: "github::create_pull_request" @@ -720,10 +724,14 @@ judge: failure: # If ANY rule matches → immediate fail - output_contains: any: ["no changes needed", "code is correct"] + - output_matches: + any: ["BEGIN PRIVATE KEY"] ``` > **Evaluation order:** `failure` outranks `success`. If any `failure` rule matches, the case fails immediately. Otherwise every `success` rule must pass. +`output_contains` performs literal substring checks. Use `output_matches` when you need regular expressions; patterns use Go `regexp` syntax and invalid patterns fail config validation. + ### judge: script — custom script Run your own script (in any language) to grade results: diff --git a/docs/zh/guide/writing-evals.md b/docs/zh/guide/writing-evals.md index a5eae4d6..712741e3 100644 --- a/docs/zh/guide/writing-evals.md +++ b/docs/zh/guide/writing-evals.md @@ -685,6 +685,10 @@ judge: all: ["bug", "null"] # 输出必须同时包含 any: ["建议修复", "推荐更改"] # 输出至少包含一个 not: ["LGTM"] # 输出不能包含 + - output_matches: # 使用 Go regexp 匹配最终输出 + all: ["(?m)^## Status$", "(?m)^## Evidence$"] + any: ["(?i)pass", "(?i)success"] + not: ["(?i)api[_-]?key\\s*="] - exit_code: 0 # 退出码必须为 0 - tool_called: # Agent 必须调用了某个工具 name: "github::create_pull_request" @@ -693,10 +697,14 @@ judge: failure: # 任一条件匹配则立即失败 - output_contains: any: ["无需修改", "代码正确"] + - output_matches: + any: ["BEGIN PRIVATE KEY"] ``` > **评估逻辑**：`failure` 优先于 `success`。任何一条 `failure` 规则匹配即立即失败；否则所有 `success` 规则必须全部满足才算通过。 +`output_contains` 执行字面 substring 检查。需要正则表达式时使用 `output_matches`；pattern 使用 Go `regexp` 语法，非法 pattern 会在配置校验阶段失败。 + ### judge: script — 自定义脚本评估 用你自己的脚本（任何语言）来评估结果： diff --git a/internal/config/schema.go b/internal/config/schema.go index 5374347c..39f19a50 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -196,6 +196,7 @@ type JudgeContextAttachment struct { // Rule is a single assertion rule for rule_based evaluation. type Rule struct { OutputContains *OutputContainsRule `json:"output_contains,omitempty" yaml:"output_contains,omitempty"` + OutputMatches *OutputMatchesRule `json:"output_matches,omitempty" yaml:"output_matches,omitempty"` ExitCode *int `json:"exit_code,omitempty" yaml:"exit_code,omitempty"` ToolCalled *ToolCalledRule `json:"tool_called,omitempty" yaml:"tool_called,omitempty"` FilesExist []string `json:"files_exist,omitempty" yaml:"files_exist,omitempty"` @@ -239,6 +240,13 @@ type OutputContainsRule struct { Not []string `json:"not,omitempty" yaml:"not,omitempty"` } +// OutputMatchesRule checks if output matches regular expressions. +type OutputMatchesRule struct { + All []string `json:"all,omitempty" yaml:"all,omitempty"` + Any []string `json:"any,omitempty" yaml:"any,omitempty"` + Not []string `json:"not,omitempty" yaml:"not,omitempty"` +} + // ToolCalledRule checks for MCP tool calls. type ToolCalledRule struct { Name string `json:"name" yaml:"name"` diff --git a/internal/config/validator.go b/internal/config/validator.go index 0d06194b..51132bee 100644 --- a/internal/config/validator.go +++ b/internal/config/validator.go @@ -235,7 +235,7 @@ func validateCaseMCP(mcpCfg MCPConfig) []string { } func judgeNeedsLocalValidation(judge JudgeConfig) bool { - return judge.Type != "" || len(judge.Skills) > 0 || judge.Context != nil + return judge.Type != "" || len(judge.Skills) > 0 || judge.Context != nil || len(judge.Success) > 0 || len(judge.Failure) > 0 } // validateJudgeTypeAndLocalFields validates judge fields that do not depend on inheritance. @@ -256,10 +256,36 @@ func validateJudgeTypeAndLocalFields(judge JudgeConfig) []string { if judge.TimeoutSeconds != nil && *judge.TimeoutSeconds < 0 { errs = append(errs, "judge.timeout_seconds must be non-negative") } + errs = append(errs, validateOutputMatchesRules("judge.success", judge.Success)...) + errs = append(errs, validateOutputMatchesRules("judge.failure", judge.Failure)...) return errs } +func validateOutputMatchesRules(field string, rules []Rule) []string { + var errs []string + for i, rule := range rules { + if rule.OutputMatches == nil { + continue + } + prefix := fmt.Sprintf("%s[%d].output_matches", field, i) + errs = append(errs, validateRegexList(prefix+".all", rule.OutputMatches.All)...) + errs = append(errs, validateRegexList(prefix+".any", rule.OutputMatches.Any)...) + errs = append(errs, validateRegexList(prefix+".not", rule.OutputMatches.Not)...) + } + return errs +} + +func validateRegexList(field string, patterns []string) []string { + var errs []string + for i, pattern := range patterns { + if _, err := regexp.Compile(pattern); err != nil { + errs = append(errs, fmt.Sprintf("%s[%d] is invalid regex %q: %v", field, i, pattern, err)) + } + } + return errs +} + func validateJudgeTypeAndRequiredFields(judge JudgeConfig) []string { errs := validateJudgeTypeAndLocalFields(judge) if judge.Type == judgeTypeScript && judge.ScriptPath == "" { diff --git a/internal/config/validator_test.go b/internal/config/validator_test.go index df2615a4..4792e478 100644 --- a/internal/config/validator_test.go +++ b/internal/config/validator_test.go @@ -39,6 +39,31 @@ func TestValidator_ValidateEvalConfig(t *testing.T) { }, wantErr: false, }, + { + name: "invalid eval-level output_matches regex", + cfg: &EvalConfig{ + SchemaVersion: "v1alpha1", + Environment: Environment{Type: "none"}, + Engine: EngineConfig{ + Name: "claude_code", + Model: ModelConfig{ + Provider: "anthropic", + Name: "claude-sonnet-4-6", + }, + }, + Cases: CasesConfig{ + Files: []string{"evals/cases/test.yaml"}, + }, + Judge: JudgeConfig{ + Type: "rule_based", + Success: []Rule{{ + OutputMatches: &OutputMatchesRule{Not: []string{"["}}, + }}, + }, + }, + wantErr: true, + errMsg: "judge.success[0].output_matches.not[0] is invalid regex", + }, { name: "valid config with opensandbox runtime image", cfg: &EvalConfig{ @@ -803,6 +828,63 @@ func TestValidator_ValidateCaseConfig(t *testing.T) { }, wantErr: false, }, + { + name: "valid output_matches rules", + cfg: &CaseConfig{ + ID: "test-case", + Input: Input{ + Prompt: "Say hello", + }, + Judge: JudgeConfig{ + Type: "rule_based", + Success: []Rule{{ + OutputMatches: &OutputMatchesRule{ + All: []string{`(?m)^## Status$`}, + Any: []string{`(?i)pass`, `(?i)success`}, + Not: []string{`(?i)api[_-]?key\s*=`}, + }, + }}, + Failure: []Rule{{ + OutputMatches: &OutputMatchesRule{Any: []string{`BEGIN PRIVATE KEY`}}, + }}, + }, + }, + wantErr: false, + }, + { + name: "invalid output_matches success regex", + cfg: &CaseConfig{ + ID: "test-case", + Input: Input{ + Prompt: "Say hello", + }, + Judge: JudgeConfig{ + Type: "rule_based", + Success: []Rule{{ + OutputMatches: &OutputMatchesRule{All: []string{"["}}, + }}, + }, + }, + wantErr: true, + errMsg: "judge.success[0].output_matches.all[0] is invalid regex", + }, + { + name: "invalid output_matches failure regex", + cfg: &CaseConfig{ + ID: "test-case", + Input: Input{ + Prompt: "Say hello", + }, + Judge: JudgeConfig{ + Type: "rule_based", + Failure: []Rule{{ + OutputMatches: &OutputMatchesRule{Any: []string{"["}}, + }}, + }, + }, + wantErr: true, + errMsg: "judge.failure[0].output_matches.any[0] is invalid regex", + }, { name: "missing prompt and turns", cfg: &CaseConfig{ diff --git a/internal/judge/README.md b/internal/judge/README.md index ff5be00e..4df1a6f5 100644 --- a/internal/judge/README.md +++ b/internal/judge/README.md @@ -10,7 +10,7 @@ All grading implementations share a unified `Judge` interface that takes an `Inp | `judge.go` | Core interface (`Judge`) and shared data types (`Input`, `Result`, `AssertionResult`, `Status`); path safety validation (`safePath`); shared file-check helper (`fileExistsInWorkspace`) | | `expect.go` | **Expect pre-check** — 7 lightweight checks; on failure short-circuits and skips the subsequent Judge to save tokens | | `factory.go` | **Factory function** — creates concrete Judge instances from `JudgeConfig`; configuration merge logic | -| `rule_based.go` | **RuleBasedJudge** — declarative rule evaluation with 5 rule types and failure-priority semantics | +| `rule_based.go` | **RuleBasedJudge** — declarative rule evaluation with failure-priority semantics | | `script.go` | **ScriptJudge** — runs an external script (exit 0 = PASS), supports timeout control | | `agent_judge.go` | **AgentJudge** — LLM-as-Judge; uses `agent.Agent` + `runtime.Runtime`; supports `pass_threshold` | @@ -169,11 +169,12 @@ Runner entry point ### RuleBasedJudge (`rule_based.go`) -5 rule assertions: +Rule assertions: | Rule | Description | |---|---| | `output_contains` | Check the final output (supports `all` / `any` / `not` modes) | +| `output_matches` | Check the final output with Go regular expressions (supports `all` / `any` / `not` modes) | | `exit_code` | Check the exit code | | `tool_called` | Check whether a tool was invoked (supports partial argument matching) | | `turn_response_contains` | *(merged into `output_contains.all`)* | diff --git a/internal/judge/rule_based.go b/internal/judge/rule_based.go index 467b15b6..d1849942 100644 --- a/internal/judge/rule_based.go +++ b/internal/judge/rule_based.go @@ -3,6 +3,7 @@ package judge import ( "context" "fmt" + "regexp" "strings" "github.com/alibaba/skill-up/internal/config" @@ -76,6 +77,8 @@ func evaluateAssertion(rule config.Rule, in Input) AssertionResult { switch { case rule.OutputContains != nil: return evalOutputContains(rule.OutputContains, in.FinalMessage) + case rule.OutputMatches != nil: + return evalOutputMatches(rule.OutputMatches, in.FinalMessage) case rule.ExitCode != nil: return evalExitCode(*rule.ExitCode, in.ExitCode) case rule.ToolCalled != nil: @@ -174,6 +177,119 @@ func evalOutputContains(rule *config.OutputContainsRule, finalMessage string) As } } +// evalOutputMatches checks the final output for regular expressions (all/any/not). +func evalOutputMatches(rule *config.OutputMatchesRule, finalMessage string) AssertionResult { + missing, invalid := missingRegexMatches("output_matches.all", rule.All, finalMessage) + if invalid != nil { + return *invalid + } + if len(missing) > 0 { + return AssertionResult{ + Text: fmt.Sprintf("output_matches.all: missing %v", missing), + Passed: false, + Evidence: fmt.Sprintf("output does not match required regex patterns: %v", missing), + } + } + + if len(rule.Any) > 0 { + found, invalid := anyRegexMatches("output_matches.any", rule.Any, finalMessage) + if invalid != nil { + return *invalid + } + if !found { + return AssertionResult{ + Text: fmt.Sprintf("output_matches.any: %v", rule.Any), + Passed: false, + Evidence: fmt.Sprintf("output does not match any of %v", rule.Any), + } + } + } + + forbiddenPattern, matched, invalid := firstMatchingRegex("output_matches.not", rule.Not, finalMessage) + if invalid != nil { + return *invalid + } + if matched { + return AssertionResult{ + Text: fmt.Sprintf("output_matches.not: %q", forbiddenPattern), + Passed: false, + Evidence: fmt.Sprintf("output matches forbidden regex pattern %q", forbiddenPattern), + } + } + + var descParts []string + if len(rule.All) > 0 { + descParts = append(descParts, fmt.Sprintf("all:%v", rule.All)) + } + if len(rule.Any) > 0 { + descParts = append(descParts, fmt.Sprintf("any:%v", rule.Any)) + } + if len(rule.Not) > 0 { + descParts = append(descParts, fmt.Sprintf("not:%v", rule.Not)) + } + desc := "output_matches" + if len(descParts) > 0 { + desc = fmt.Sprintf("output_matches{%s}", strings.Join(descParts, ", ")) + } + + return AssertionResult{ + Text: desc, + Passed: true, + Evidence: fmt.Sprintf("output satisfies all regex checks (%s)", strings.Join(descParts, ", ")), + } +} + +func missingRegexMatches(field string, patterns []string, finalMessage string) ([]string, *AssertionResult) { + var missing []string + for _, pattern := range patterns { + matched, err := regexp.MatchString(pattern, finalMessage) + if err != nil { + invalid := invalidRegexResult(field, pattern, err) + return nil, &invalid + } + if !matched { + missing = append(missing, pattern) + } + } + return missing, nil +} + +func anyRegexMatches(field string, patterns []string, finalMessage string) (bool, *AssertionResult) { + for _, pattern := range patterns { + matched, err := regexp.MatchString(pattern, finalMessage) + if err != nil { + invalid := invalidRegexResult(field, pattern, err) + return false, &invalid + } + if matched { + return true, nil + } + } + return false, nil +} + +func firstMatchingRegex(field string, patterns []string, finalMessage string) (string, bool, *AssertionResult) { + for _, pattern := range patterns { + matched, err := regexp.MatchString(pattern, finalMessage) + if err != nil { + invalid := invalidRegexResult(field, pattern, err) + return "", false, &invalid + } + if matched { + return pattern, true, nil + } + } + return "", false, nil +} + +func invalidRegexResult(field, pattern string, err error) AssertionResult { + return AssertionResult{ + Text: fmt.Sprintf("%s: invalid %q", field, pattern), + Passed: false, + Evidence: fmt.Sprintf("invalid regex pattern %q: %v", pattern, err), + } +} + // evalExitCode checks that the exit code matches. func evalExitCode(expected, actual int) AssertionResult { if actual == expected { diff --git a/internal/judge/rule_based_test.go b/internal/judge/rule_based_test.go index c00c0412..a353e0fc 100644 --- a/internal/judge/rule_based_test.go +++ b/internal/judge/rule_based_test.go @@ -92,6 +92,113 @@ func TestRuleBased_OutputContains_Combined(t *testing.T) { assertStatus(t, r, StatusPass) } +// --------------------------------------------------------------------------- +// output_matches +// --------------------------------------------------------------------------- + +func TestRuleBased_OutputMatches_All_Pass(t *testing.T) { + j := NewRuleBasedJudge(config.JudgeConfig{ + Success: []config.Rule{{ + OutputMatches: &config.OutputMatchesRule{All: []string{`(?m)^## Status$`, `(?m)^## Evidence$`}}, + }}, + }) + r, err := j.Evaluate(context.Background(), Input{FinalMessage: "## Status\npass\n## Evidence\nverified"}) + assertNoError(t, err) + assertStatus(t, r, StatusPass) +} + +func TestRuleBased_OutputMatches_All_Fail(t *testing.T) { + j := NewRuleBasedJudge(config.JudgeConfig{ + Success: []config.Rule{{ + OutputMatches: &config.OutputMatchesRule{All: []string{`(?m)^## Status$`, `(?m)^## Evidence$`}}, + }}, + }) + r, err := j.Evaluate(context.Background(), Input{FinalMessage: "## Status\npass"}) + assertNoError(t, err) + assertStatus(t, r, StatusFail) +} + +func TestRuleBased_OutputMatches_Any_Pass(t *testing.T) { + j := NewRuleBasedJudge(config.JudgeConfig{ + Success: []config.Rule{{ + OutputMatches: &config.OutputMatchesRule{Any: []string{`(?i)pass`, `(?i)success`}}, + }}, + }) + r, err := j.Evaluate(context.Background(), Input{FinalMessage: "The check ended in SUCCESS"}) + assertNoError(t, err) + assertStatus(t, r, StatusPass) +} + +func TestRuleBased_OutputMatches_Any_Fail(t *testing.T) { + j := NewRuleBasedJudge(config.JudgeConfig{ + Success: []config.Rule{{ + OutputMatches: &config.OutputMatchesRule{Any: []string{`(?i)pass`, `(?i)success`}}, + }}, + }) + r, err := j.Evaluate(context.Background(), Input{FinalMessage: "The check failed"}) + assertNoError(t, err) + assertStatus(t, r, StatusFail) +} + +func TestRuleBased_OutputMatches_Not_Pass(t *testing.T) { + j := NewRuleBasedJudge(config.JudgeConfig{ + Success: []config.Rule{{ + OutputMatches: &config.OutputMatchesRule{Not: []string{`(?i)api[_-]?key\s*=`, `BEGIN PRIVATE KEY`}}, + }}, + }) + r, err := j.Evaluate(context.Background(), Input{FinalMessage: "No secrets here"}) + assertNoError(t, err) + assertStatus(t, r, StatusPass) +} + +func TestRuleBased_OutputMatches_Not_Fail(t *testing.T) { + j := NewRuleBasedJudge(config.JudgeConfig{ + Success: []config.Rule{{ + OutputMatches: &config.OutputMatchesRule{Not: []string{`(?i)api[_-]?key\s*=`}}, + }}, + }) + r, err := j.Evaluate(context.Background(), Input{FinalMessage: "api_key = leaked"}) + assertNoError(t, err) + assertStatus(t, r, StatusFail) +} + +func TestRuleBased_OutputMatches_Not_EmptyRegexFails(t *testing.T) { + j := NewRuleBasedJudge(config.JudgeConfig{ + Success: []config.Rule{{ + OutputMatches: &config.OutputMatchesRule{Not: []string{""}}, + }}, + }) + r, err := j.Evaluate(context.Background(), Input{FinalMessage: "anything"}) + assertNoError(t, err) + assertStatus(t, r, StatusFail) +} + +func TestRuleBased_OutputMatches_Combined(t *testing.T) { + j := NewRuleBasedJudge(config.JudgeConfig{ + Success: []config.Rule{{ + OutputMatches: &config.OutputMatchesRule{ + All: []string{`(?m)^## Status$`}, + Any: []string{`(?i)pass`, `(?i)success`}, + Not: []string{`(?i)api[_-]?key\s*=`}, + }, + }}, + }) + r, err := j.Evaluate(context.Background(), Input{FinalMessage: "## Status\nPASS"}) + assertNoError(t, err) + assertStatus(t, r, StatusPass) +} + +func TestRuleBased_OutputMatches_EmptyGroups_Pass(t *testing.T) { + j := NewRuleBasedJudge(config.JudgeConfig{ + Success: []config.Rule{{ + OutputMatches: &config.OutputMatchesRule{}, + }}, + }) + r, err := j.Evaluate(context.Background(), Input{FinalMessage: "anything"}) + assertNoError(t, err) + assertStatus(t, r, StatusPass) +} + // --------------------------------------------------------------------------- // exit_code // --------------------------------------------------------------------------- @@ -311,6 +418,20 @@ func TestRuleBased_FailureRules_TakePriority(t *testing.T) { assertStatus(t, r, StatusFail) } +func TestRuleBased_FailureRules_OutputMatches_TakePriority(t *testing.T) { + j := NewRuleBasedJudge(config.JudgeConfig{ + Success: []config.Rule{{ + OutputMatches: &config.OutputMatchesRule{All: []string{`(?i)pass`}}, + }}, + Failure: []config.Rule{{ + OutputMatches: &config.OutputMatchesRule{Any: []string{`(?i)api[_-]?key\s*=`}}, + }}, + }) + r, err := j.Evaluate(context.Background(), Input{FinalMessage: "PASS\napi_key = leaked"}) + assertNoError(t, err) + assertStatus(t, r, StatusFail) +} + func TestRuleBased_FailureRules_NoMatch_SuccessEvaluated(t *testing.T) { j := NewRuleBasedJudge(config.JudgeConfig{ Success: []config.Rule{{ diff --git a/skills/skill-upper/references/judge-types.md b/skills/skill-upper/references/judge-types.md index 8dfe9685..adc8a502 100644 --- a/skills/skill-upper/references/judge-types.md +++ b/skills/skill-upper/references/judge-types.md @@ -20,6 +20,10 @@ judge: all: ["bug", "null"] any: ["建议修复", "推荐更改"] not: ["LGTM"] + - output_matches: + all: ["(?m)^## Status$", "(?m)^## Evidence$"] + any: ["(?i)pass", "(?i)success"] + not: ["(?i)api[_-]?key\\s*="] - exit_code: 0 - tool_called: name: "github::create_pull_request" @@ -35,6 +39,7 @@ judge: **支持的匹配器**： - `output_contains` +- `output_matches`（Go regexp，支持 `all` / `any` / `not`） - `exit_code` - `tool_called` - `files_exist` / `files_not_exist`