feat(judge): add output_matches rule

docs/guide/writing-evals.md

@@ -712,6 +712,10 @@ judge:
         all: ["bug", "null"]                  # Must contain ALL
         any: ["suggest fix", "recommend"]     # Must contain at least one
         not: ["LGTM"]                         # Must NOT contain
+    - output_matches:                         # Go regexp patterns against final output
+        all: ["(?m)^## Status$", "(?m)^## Evidence$"]
+        any: ["(?i)pass", "(?i)success"]
+        not: ["(?i)api[_-]?key\\s*="]
     - exit_code: 0
     - tool_called:                            # Agent must invoke this tool
         name: "github::create_pull_request"
@@ -720,10 +724,14 @@ judge:
   failure:                                    # If ANY rule matches → immediate fail
     - output_contains:
         any: ["no changes needed", "code is correct"]
+    - output_matches:
+        any: ["BEGIN PRIVATE KEY"]
 ```
 
 > **Evaluation order:** `failure` outranks `success`. If any `failure` rule matches, the case fails immediately. Otherwise every `success` rule must pass.
 
+`output_contains` performs literal substring checks. Use `output_matches` when you need regular expressions; patterns use Go `regexp` syntax and invalid patterns fail config validation.
+
 ### judge: script — custom script
 
 Run your own script (in any language) to grade results:

docs/zh/guide/writing-evals.md

@@ -685,6 +685,10 @@ judge:
         all: ["bug", "null"]                  # 输出必须同时包含
         any: ["建议修复", "推荐更改"]          # 输出至少包含一个
         not: ["LGTM"]                         # 输出不能包含
+    - output_matches:                         # 使用 Go regexp 匹配最终输出
+        all: ["(?m)^## Status$", "(?m)^## Evidence$"]
+        any: ["(?i)pass", "(?i)success"]
+        not: ["(?i)api[_-]?key\\s*="]
     - exit_code: 0                            # 退出码必须为 0
     - tool_called:                            # Agent 必须调用了某个工具
         name: "github::create_pull_request"
@@ -693,10 +697,14 @@ judge:
   failure:                                    # 任一条件匹配则立即失败
     - output_contains:
         any: ["无需修改", "代码正确"]
+    - output_matches:
+        any: ["BEGIN PRIVATE KEY"]
 ```
 
 > **评估逻辑**：`failure` 优先于 `success`。任何一条 `failure` 规则匹配即立即失败；否则所有 `success` 规则必须全部满足才算通过。
 
+`output_contains` 执行字面 substring 检查。需要正则表达式时使用 `output_matches`；pattern 使用 Go `regexp` 语法，非法 pattern 会在配置校验阶段失败。
+
 ### judge: script — 自定义脚本评估
 
 用你自己的脚本（任何语言）来评估结果：

internal/config/schema.go

@@ -196,6 +196,7 @@ type JudgeContextAttachment struct {
 // Rule is a single assertion rule for rule_based evaluation.
 type Rule struct {
 	OutputContains          *OutputContainsRule          `json:"output_contains,omitempty"            yaml:"output_contains,omitempty"`
+	OutputMatches           *OutputMatchesRule           `json:"output_matches,omitempty"             yaml:"output_matches,omitempty"`
 	ExitCode                *int                         `json:"exit_code,omitempty"                  yaml:"exit_code,omitempty"`
 	ToolCalled              *ToolCalledRule              `json:"tool_called,omitempty"                yaml:"tool_called,omitempty"`
 	FilesExist              []string                     `json:"files_exist,omitempty"                yaml:"files_exist,omitempty"`
@@ -239,6 +240,13 @@ type OutputContainsRule struct {
 	Not []string `json:"not,omitempty" yaml:"not,omitempty"`
 }
 
+// OutputMatchesRule checks if output matches regular expressions.
+type OutputMatchesRule struct {
+	All []string `json:"all,omitempty" yaml:"all,omitempty"`
+	Any []string `json:"any,omitempty" yaml:"any,omitempty"`
+	Not []string `json:"not,omitempty" yaml:"not,omitempty"`
+}
+
 // ToolCalledRule checks for MCP tool calls.
 type ToolCalledRule struct {
 	Name string         `json:"name"           yaml:"name"`

internal/config/validator.go

@@ -235,7 +235,7 @@ func validateCaseMCP(mcpCfg MCPConfig) []string {
 }
 
 func judgeNeedsLocalValidation(judge JudgeConfig) bool {
-	return judge.Type != "" || len(judge.Skills) > 0 || judge.Context != nil
+	return judge.Type != "" || len(judge.Skills) > 0 || judge.Context != nil || len(judge.Success) > 0 || len(judge.Failure) > 0
 }
 
 // validateJudgeTypeAndLocalFields validates judge fields that do not depend on inheritance.
@@ -256,10 +256,36 @@ func validateJudgeTypeAndLocalFields(judge JudgeConfig) []string {
 	if judge.TimeoutSeconds != nil && *judge.TimeoutSeconds < 0 {
 		errs = append(errs, "judge.timeout_seconds must be non-negative")
 	}
+	errs = append(errs, validateOutputMatchesRules("judge.success", judge.Success)...)
+	errs = append(errs, validateOutputMatchesRules("judge.failure", judge.Failure)...)
 
 	return errs
 }
 
+func validateOutputMatchesRules(field string, rules []Rule) []string {
+	var errs []string
+	for i, rule := range rules {
+		if rule.OutputMatches == nil {
+			continue
+		}
+		prefix := fmt.Sprintf("%s[%d].output_matches", field, i)
+		errs = append(errs, validateRegexList(prefix+".all", rule.OutputMatches.All)...)
+		errs = append(errs, validateRegexList(prefix+".any", rule.OutputMatches.Any)...)
+		errs = append(errs, validateRegexList(prefix+".not", rule.OutputMatches.Not)...)
+	}
+	return errs
+}
+
+func validateRegexList(field string, patterns []string) []string {
+	var errs []string
+	for i, pattern := range patterns {
+		if _, err := regexp.Compile(pattern); err != nil {
+			errs = append(errs, fmt.Sprintf("%s[%d] is invalid regex %q: %v", field, i, pattern, err))
+		}
+	}
+	return errs
+}
+
 func validateJudgeTypeAndRequiredFields(judge JudgeConfig) []string {
 	errs := validateJudgeTypeAndLocalFields(judge)
 	if judge.Type == judgeTypeScript && judge.ScriptPath == "" {

internal/config/validator_test.go

@@ -39,6 +39,31 @@ func TestValidator_ValidateEvalConfig(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "invalid eval-level output_matches regex",
+			cfg: &EvalConfig{
+				SchemaVersion: "v1alpha1",
+				Environment:   Environment{Type: "none"},
+				Engine: EngineConfig{
+					Name: "claude_code",
+					Model: ModelConfig{
+						Provider: "anthropic",
+						Name:     "claude-sonnet-4-6",
+					},
+				},
+				Cases: CasesConfig{
+					Files: []string{"evals/cases/test.yaml"},
+				},
+				Judge: JudgeConfig{
+					Type: "rule_based",
+					Success: []Rule{{
+						OutputMatches: &OutputMatchesRule{Not: []string{"["}},
+					}},
+				},
+			},
+			wantErr: true,
+			errMsg:  "judge.success[0].output_matches.not[0] is invalid regex",
+		},
 		{
 			name: "valid config with opensandbox runtime image",
 			cfg: &EvalConfig{
@@ -803,6 +828,63 @@ func TestValidator_ValidateCaseConfig(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "valid output_matches rules",
+			cfg: &CaseConfig{
+				ID: "test-case",
+				Input: Input{
+					Prompt: "Say hello",
+				},
+				Judge: JudgeConfig{
+					Type: "rule_based",
+					Success: []Rule{{
+						OutputMatches: &OutputMatchesRule{
+							All: []string{`(?m)^## Status$`},
+							Any: []string{`(?i)pass`, `(?i)success`},
+							Not: []string{`(?i)api[_-]?key\s*=`},
+						},
+					}},
+					Failure: []Rule{{
+						OutputMatches: &OutputMatchesRule{Any: []string{`BEGIN PRIVATE KEY`}},
+					}},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "invalid output_matches success regex",
+			cfg: &CaseConfig{
+				ID: "test-case",
+				Input: Input{
+					Prompt: "Say hello",
+				},
+				Judge: JudgeConfig{
+					Type: "rule_based",
+					Success: []Rule{{
+						OutputMatches: &OutputMatchesRule{All: []string{"["}},
+					}},
+				},
+			},
+			wantErr: true,
+			errMsg:  "judge.success[0].output_matches.all[0] is invalid regex",
+		},
+		{
+			name: "invalid output_matches failure regex",
+			cfg: &CaseConfig{
+				ID: "test-case",
+				Input: Input{
+					Prompt: "Say hello",
+				},
+				Judge: JudgeConfig{
+					Type: "rule_based",
+					Failure: []Rule{{
+						OutputMatches: &OutputMatchesRule{Any: []string{"["}},
+					}},
+				},
+			},
+			wantErr: true,
+			errMsg:  "judge.failure[0].output_matches.any[0] is invalid regex",
+		},
 		{
 			name: "missing prompt and turns",
 			cfg: &CaseConfig{

internal/judge/README.md

@@ -10,7 +10,7 @@ All grading implementations share a unified `Judge` interface that takes an `Inp
 | `judge.go` | Core interface (`Judge`) and shared data types (`Input`, `Result`, `AssertionResult`, `Status`); path safety validation (`safePath`); shared file-check helper (`fileExistsInWorkspace`) |
 | `expect.go` | **Expect pre-check** — 7 lightweight checks; on failure short-circuits and skips the subsequent Judge to save tokens |
 | `factory.go` | **Factory function** — creates concrete Judge instances from `JudgeConfig`; configuration merge logic |
-| `rule_based.go` | **RuleBasedJudge** — declarative rule evaluation with 5 rule types and failure-priority semantics |
+| `rule_based.go` | **RuleBasedJudge** — declarative rule evaluation with failure-priority semantics |
 | `script.go` | **ScriptJudge** — runs an external script (exit 0 = PASS), supports timeout control |
 | `agent_judge.go` | **AgentJudge** — LLM-as-Judge; uses `agent.Agent` + `runtime.Runtime`; supports `pass_threshold` |
 
@@ -169,11 +169,12 @@ Runner entry point
 
 ### RuleBasedJudge (`rule_based.go`)
 
-5 rule assertions:
+Rule assertions:
 
 | Rule | Description |
 |---|---|
 | `output_contains` | Check the final output (supports `all` / `any` / `not` modes) |
+| `output_matches` | Check the final output with Go regular expressions (supports `all` / `any` / `not` modes) |
 | `exit_code` | Check the exit code |
 | `tool_called` | Check whether a tool was invoked (supports partial argument matching) |
 | `turn_response_contains` | *(merged into `output_contains.all`)* |

internal/judge/rule_based.go

@@ -3,6 +3,7 @@ package judge
 import (
 	"context"
 	"fmt"
+	"regexp"
 	"strings"
 
 	"github.com/alibaba/skill-up/internal/config"
@@ -76,6 +77,8 @@ func evaluateAssertion(rule config.Rule, in Input) AssertionResult {
 	switch {
 	case rule.OutputContains != nil:
 		return evalOutputContains(rule.OutputContains, in.FinalMessage)
+	case rule.OutputMatches != nil:
+		return evalOutputMatches(rule.OutputMatches, in.FinalMessage)
 	case rule.ExitCode != nil:
 		return evalExitCode(*rule.ExitCode, in.ExitCode)
 	case rule.ToolCalled != nil:
@@ -174,6 +177,119 @@ func evalOutputContains(rule *config.OutputContainsRule, finalMessage string) As
 	}
 }
 
+// evalOutputMatches checks the final output for regular expressions (all/any/not).
+func evalOutputMatches(rule *config.OutputMatchesRule, finalMessage string) AssertionResult {
+	missing, invalid := missingRegexMatches("output_matches.all", rule.All, finalMessage)
+	if invalid != nil {
+		return *invalid
+	}
+	if len(missing) > 0 {
+		return AssertionResult{
+			Text:     fmt.Sprintf("output_matches.all: missing %v", missing),
+			Passed:   false,
+			Evidence: fmt.Sprintf("output does not match required regex patterns: %v", missing),
+		}
+	}
+
+	if len(rule.Any) > 0 {
+		found, invalid := anyRegexMatches("output_matches.any", rule.Any, finalMessage)
+		if invalid != nil {
+			return *invalid
+		}
+		if !found {
+			return AssertionResult{
+				Text:     fmt.Sprintf("output_matches.any: %v", rule.Any),
+				Passed:   false,
+				Evidence: fmt.Sprintf("output does not match any of %v", rule.Any),
+			}
+		}
+	}
+
+	forbiddenPattern, matched, invalid := firstMatchingRegex("output_matches.not", rule.Not, finalMessage)
+	if invalid != nil {
+		return *invalid
+	}
+	if matched {
+		return AssertionResult{
+			Text:     fmt.Sprintf("output_matches.not: %q", forbiddenPattern),
+			Passed:   false,
+			Evidence: fmt.Sprintf("output matches forbidden regex pattern %q", forbiddenPattern),
+		}
+	}
+
+	var descParts []string
+	if len(rule.All) > 0 {
+		descParts = append(descParts, fmt.Sprintf("all:%v", rule.All))
+	}
+	if len(rule.Any) > 0 {
+		descParts = append(descParts, fmt.Sprintf("any:%v", rule.Any))
+	}
+	if len(rule.Not) > 0 {
+		descParts = append(descParts, fmt.Sprintf("not:%v", rule.Not))
+	}
+	desc := "output_matches"
+	if len(descParts) > 0 {
+		desc = fmt.Sprintf("output_matches{%s}", strings.Join(descParts, ", "))
+	}
+
+	return AssertionResult{
+		Text:     desc,
+		Passed:   true,
+		Evidence: fmt.Sprintf("output satisfies all regex checks (%s)", strings.Join(descParts, ", ")),
+	}
+}
+
+func missingRegexMatches(field string, patterns []string, finalMessage string) ([]string, *AssertionResult) {
+	var missing []string
+	for _, pattern := range patterns {
+		matched, err := regexp.MatchString(pattern, finalMessage)
+		if err != nil {
+			invalid := invalidRegexResult(field, pattern, err)
+			return nil, &invalid
+		}
+		if !matched {
+			missing = append(missing, pattern)
+		}
+	}
+	return missing, nil
+}
+
+func anyRegexMatches(field string, patterns []string, finalMessage string) (bool, *AssertionResult) {
+	for _, pattern := range patterns {
+		matched, err := regexp.MatchString(pattern, finalMessage)
+		if err != nil {
+			invalid := invalidRegexResult(field, pattern, err)
+			return false, &invalid
+		}
+		if matched {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func firstMatchingRegex(field string, patterns []string, finalMessage string) (string, bool, *AssertionResult) {
+	for _, pattern := range patterns {
+		matched, err := regexp.MatchString(pattern, finalMessage)
+		if err != nil {
+			invalid := invalidRegexResult(field, pattern, err)
+			return "", false, &invalid
+		}
+		if matched {
+			return pattern, true, nil
+		}
+	}
+	return "", false, nil
+}
+
+func invalidRegexResult(field, pattern string, err error) AssertionResult {
+	return AssertionResult{
+		Text:     fmt.Sprintf("%s: invalid %q", field, pattern),
+		Passed:   false,
+		Evidence: fmt.Sprintf("invalid regex pattern %q: %v", pattern, err),
+	}
+}
+
 // evalExitCode checks that the exit code matches.
 func evalExitCode(expected, actual int) AssertionResult {
 	if actual == expected {