fix(redaction): strip C0/C1 bytes before shape matching

internal/redaction/redaction.go

@@ -9,6 +9,7 @@ import (
 	"sort"
 	"strings"
 	"unicode"
+	"unicode/utf8"
 )
 
 const (
@@ -68,12 +69,36 @@ var sensitiveKeys = map[string]struct{}{
 	"zero_api_key":          {},
 }
 
+// ctrlGap matches C0/C1 bytes (Cc other than tab/LF/CR, plus lone Latin-1 C1)
+// between characters of a secret shape. Matching stays on the original string:
+// a deleted control is never a join, so \b still treats wordchar+control as a
+// boundary and tokens that were never adjacent stay that way. Tab/LF/CR are
+// excluded so log line structure is unchanged. \x{FFFD} is how Go's regexp
+// engine reports a lone invalid UTF-8 C1 byte such as 0x9B.
+const ctrlGap = `[\x00-\x08\x0b\x0c\x0e-\x1f\x7f\x80-\x9f\x{FFFD}]*`
+
+// ctrlLit quotes s as a regexp literal with ctrlGap after every rune, so a
+// NUL/ESC/C1 may split the literal without breaking the match.
+func ctrlLit(s string) string {
+	var b strings.Builder
+	b.Grow(len(s) * (1 + len(ctrlGap)))
+	for _, r := range s {
+		b.WriteString(regexp.QuoteMeta(string(r)))
+		b.WriteString(ctrlGap)
+	}
+	return b.String()
+}
+
+func secretBody(class, quant string) string {
+	return `(?:` + class + ctrlGap + `)` + quant
+}
+
 // openaiKeyPattern mirrors secrets.Scan's broad sk- body. Known OpenAI
 // prefixes (sk-proj-/sk-svcacct-/sk-admin-) are always redacted; other sk-
 // digit-free matches with an interior hyphen are left alone (kebab-case false
 // positives), while digit-free legacy sk- credentials are still redacted.
 // Applied via ReplaceAllStringFunc rather than the plain list below.
-var openaiKeyPattern = regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{20,}`)
+var openaiKeyPattern = regexp.MustCompile(`\b` + ctrlLit("sk-") + secretBody(`[A-Za-z0-9_-]`, `{20,}`))
 
 // textSecretPatterns mirror secrets.Scan for end-boundary behavior and the
 // shared high-confidence shapes. A leading \b keeps each pattern from firing
@@ -84,16 +109,18 @@ var openaiKeyPattern = regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{20,}`)
 // (not in secrets.Scan); ASIA temporary access keys are kept alongside AKIA.
 // openai keys are handled separately (digit filter). JWT has a strict form
 // (both segments start with eyJ) and a looser three-segment form.
+// ctrlGap between shape characters keeps NUL/ESC/C1 split secrets matching
+// without stripping those bytes out of the subject first.
 var textSecretPatterns = []*regexp.Regexp{
-	regexp.MustCompile(`\bsk-ant-(?:api\d{2}-)?[A-Za-z0-9_-]{20,}`),
-	regexp.MustCompile(`\bgithub_pat_[A-Za-z0-9_]{22,}`),
-	regexp.MustCompile(`\bgh[pousr]_[A-Za-z0-9]{36,}`),
-	regexp.MustCompile(`\bglpat-[A-Za-z0-9_-]{12,}`),
-	regexp.MustCompile(`\bAIza[0-9A-Za-z\-_]{35,}`),
-	regexp.MustCompile(`\bxox[baprs]-[A-Za-z0-9-]{10,}`),
-	regexp.MustCompile(`\b(?:AKIA|ASIA)[A-Z0-9]{16}`),
-	regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}`),
-	regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}`),
+	regexp.MustCompile(`\b` + ctrlLit("sk-ant-") + `(?:` + ctrlLit("api") + `\d` + ctrlGap + `\d` + ctrlGap + ctrlLit("-") + `)?` + secretBody(`[A-Za-z0-9_-]`, `{20,}`)),
+	regexp.MustCompile(`\b` + ctrlLit("github_pat_") + secretBody(`[A-Za-z0-9_]`, `{22,}`)),
+	regexp.MustCompile(`\b` + ctrlLit("gh") + `[pousr]` + ctrlGap + `_` + ctrlGap + secretBody(`[A-Za-z0-9]`, `{36,}`)),
+	regexp.MustCompile(`\b` + ctrlLit("glpat-") + secretBody(`[A-Za-z0-9_-]`, `{12,}`)),
+	regexp.MustCompile(`\b` + ctrlLit("AIza") + secretBody(`[0-9A-Za-z\-_]`, `{35,}`)),
+	regexp.MustCompile(`\b` + ctrlLit("xox") + `[baprs]` + ctrlGap + `-` + ctrlGap + secretBody(`[A-Za-z0-9-]`, `{10,}`)),
+	regexp.MustCompile(`\b(?:` + ctrlLit("AKIA") + `|` + ctrlLit("ASIA") + `)` + secretBody(`[A-Z0-9]`, `{16}`)),
+	regexp.MustCompile(`\b` + ctrlLit("eyJ") + secretBody(`[A-Za-z0-9_-]`, `{10,}`) + `\.` + ctrlGap + ctrlLit("eyJ") + secretBody(`[A-Za-z0-9_-]`, `{10,}`) + `\.` + ctrlGap + secretBody(`[A-Za-z0-9_-]`, `{10,}`)),
+	regexp.MustCompile(`\b` + ctrlLit("eyJ") + secretBody(`[A-Za-z0-9_-]`, `{10,}`) + `\.` + ctrlGap + secretBody(`[A-Za-z0-9_-]`, `{10,}`) + `\.` + ctrlGap + secretBody(`[A-Za-z0-9_-]`, `{10,}`)),
 }
 
 var (
@@ -170,8 +197,70 @@ func keyLooksSensitive(normalized string) bool {
 	return false
 }
 
+// stripControlBytes removes C0/C1 controls (Cc other than tab, LF, and CR).
+// Used to normalize an already-matched secret so prefix/digit checks see the
+// rejoined shape. It is matching-time only and must not be applied to
+// RedactString's input or return value. Tab/LF/CR stay. Lone Latin-1 C1 bytes
+// (0x80–0x9F, invalid UTF-8) are stripped too; UTF-8 continuation bytes are
+// not, because they are not controls.
+func stripControlBytes(s string) string {
+	for i := 0; i < len(s); {
+		c := s[i]
+		if c < 0x80 {
+			if c != '\t' && c != '\n' && c != '\r' && (c < 0x20 || c == 0x7F) {
+				return stripControlBytesFrom(s, i)
+			}
+			i++
+			continue
+		}
+		if c <= 0x9F {
+			// 0x80–0x9F at a rune boundary is a lone C1 byte, not UTF-8.
+			return stripControlBytesFrom(s, i)
+		}
+		r, size := utf8.DecodeRuneInString(s[i:])
+		if unicode.IsControl(r) {
+			return stripControlBytesFrom(s, i)
+		}
+		i += size
+	}
+	return s
+}
+
+func stripControlBytesFrom(s string, start int) string {
+	var b strings.Builder
+	b.Grow(len(s))
+	b.WriteString(s[:start])
+	for i := start; i < len(s); {
+		c := s[i]
+		if c < 0x80 {
+			if c != '\t' && c != '\n' && c != '\r' && (c < 0x20 || c == 0x7F) {
+				i++
+				continue
+			}
+			b.WriteByte(c)
+			i++
+			continue
+		}
+		if c <= 0x9F {
+			i++
+			continue
+		}
+		r, size := utf8.DecodeRuneInString(s[i:])
+		if unicode.IsControl(r) {
+			i += size
+			continue
+		}
+		b.WriteString(s[i : i+size])
+		i += size
+	}
+	return b.String()
+}
+
 func RedactString(value string, options Options) string {
 	replacement := replacement(options)
+	// Match on the original string. Shape patterns allow C0/C1 gaps between
+	// characters so a split secret still matches; stripping first would join
+	// tokens that were never adjacent and make \b miss a leading wordchar.
 	redacted := value
 	if len(options.ExtraSecretValues) > 0 {
 		secrets := append([]string{}, options.ExtraSecretValues...)
@@ -227,8 +316,9 @@ func RedactString(value string, options Options) string {
 	// openai keys first so the filter can drop kebab-case false positives
 	// before any other pattern rewrites nearby text.
 	redacted = openaiKeyPattern.ReplaceAllStringFunc(redacted, func(match string) string {
-		if !knownOpenAIKeyPrefix(match) && !secretMatchHasDigit(match) &&
-			strings.Contains(strings.TrimPrefix(match, "sk-"), "-") {
+		normalized := stripControlBytes(match)
+		if !knownOpenAIKeyPrefix(normalized) && !secretMatchHasDigit(normalized) &&
+			strings.Contains(strings.TrimPrefix(normalized, "sk-"), "-") {
 			return match
 		}
 		return replacement

internal/redaction/redaction_test.go

@@ -141,3 +141,87 @@ func containsCircular(v any) bool {
 	}
 	return false
 }
+
+func TestRedactStringCatchesSecretsSplitByControlBytes(t *testing.T) {
+	// Unsplit passing is not coverage: a NUL/ESC/C1 in the body splits the
+	// shape so the patterns miss it unless matching allows those controls as
+	// gaps between body characters (without joining unrelated tokens).
+	const prefix = "sk-ant-api03-"
+	const body = "abcdefghijklmnopqrstuvwxyz"
+	unsplit := prefix + body
+	if got := RedactString(unsplit, Options{}); strings.Contains(got, body) {
+		t.Fatalf("unsplit secret not redacted (test setup): %q", got)
+	}
+
+	cases := []struct {
+		name  string
+		split string
+	}{
+		{name: "NUL", split: "\x00"},
+		{name: "ESC", split: "\x1b"},
+		{name: "C1", split: "\x9b"},
+		{name: "UTF-8 C1", split: string(rune(0x9B))},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			input := prefix + tc.split + body
+			got := RedactString(input, Options{})
+			if strings.Contains(got, body) {
+				t.Fatalf("secret split by %s leaked in %q", tc.name, got)
+			}
+			if strings.Contains(got, prefix) {
+				t.Fatalf("secret prefix split by %s leaked in %q", tc.name, got)
+			}
+			if !strings.Contains(got, RedactedSecret) {
+				t.Fatalf("expected %q after %s split, got %q", RedactedSecret, tc.name, got)
+			}
+		})
+	}
+}
+
+func TestRedactStringPreservesAllowedWhitespaceAndUTF8(t *testing.T) {
+	input := "safe\tline\nnext\rfinal café"
+	if got := RedactString(input, Options{}); got != input {
+		t.Fatalf("unexpected normalization: %q", got)
+	}
+}
+
+func TestRedactStringWordcharBeforeNULAnthropicKey(t *testing.T) {
+	// Matching on a control-stripped copy joins "id42" and the key, so \b in
+	// textSecretPatterns misses and the secret leaks. Matching on the original
+	// treats the NUL as a boundary; leaked must be false.
+	const secret = "sk-ant-api03-abcdefghijklmnopqrstuvwxyz"
+	if got := RedactString(secret, Options{}); strings.Contains(got, "sk-ant-api03-") {
+		t.Fatalf("unsplit secret not redacted (test setup): %q", got)
+	}
+	input := "id42\x00" + secret
+	got := RedactString(input, Options{})
+	leaked := strings.Contains(got, secret) || strings.Contains(got, "sk-ant-api03-")
+	if leaked {
+		t.Fatalf("wordchar-before-NUL+anthropic-key leaked=true out=%q", got)
+	}
+	if !strings.Contains(got, RedactedSecret) {
+		t.Fatalf("wordchar-before-NUL+anthropic-key leaked=false want %q, got %q", RedactedSecret, got)
+	}
+}
+
+func TestRedactStringControlBytesWithoutSecretStayIdentical(t *testing.T) {
+	// scrubResultSecrets sets Result.Redacted when RedactString's result !=
+	// Output. Stripping is matching-time only: no-secret control bytes must
+	// remain byte-identical so Redacted stays false.
+	cases := []struct {
+		name  string
+		input string
+	}{
+		{name: "form feed in source", input: "package main\n\ffunc main() {}\n"},
+		{name: "Windows-1252 quotes", input: "Don\x92t \x93quote\x94 me\n"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := RedactString(tc.input, Options{})
+			if got != tc.input {
+				t.Fatalf("no-secret input not byte-identical:\n in=%q\nout=%q", tc.input, got)
+			}
+		})
+	}
+}