Milestone 12 (part): benchmarks vs cgo pg_query_go + allocation profiling (#9)

Author: kyleconroy

.gitignore

@@ -0,0 +1,2 @@
+# Built cgo oracle binary (go build ./... in oracle/)
+oracle/oracle

PLAN.md

@@ -434,7 +434,9 @@ gate; the corpus harness exists before the first line of the lexer.
     scheduled CI job; `go test -race` over the parallel corpus run;
     benchmarks vs pg_query_go (cgo) and wasilibs (wasm) — expect wins from
     no cgo crossings and true in-process parallelism; memory profiling on
-    the stress queries.
+    the stress queries. *Benchmarks + memory profiling landed 2026-08-17;
+    see "As-built notes (milestone 12, benchmarks + profiling)". Fuzzing,
+    `-race`, and the wasilibs comparison remain.*
 13. **sqlc integration** (in the sqlc repo). Replace
     `parse_default.go`/`parse_wasi.go` with one unconditional file; swap the
     import path in `convert.go` et al.; drop `wasilibs/go-pgquery`,
@@ -722,6 +724,45 @@ Measured at the pin, where the plan's estimates differ:
   `TestParsePlPgSQL` now runs green; no entry point returns
   not-implemented anymore.
 
+### As-built notes (milestone 12, benchmarks + profiling)
+
+- The root `benchmark_test.go` mirrors pg_query_go v6.2.2's benchmark file
+  name-for-name (same queries, same globals trick), with additions: JSON /
+  Scan variants and `*Stress` benchmarks over the corpus's largest input,
+  the 1.1 MB multi-VALUES INSERT (fingerprint suite, case 073).
+  `oracle/benchmark_test.go` is the cgo twin — identical names and inputs
+  against the pinned pg_query_go — so a run of each diffs directly:
+  `benchstat cgo.txt pure.txt` (after normalizing the `pkg:` line, since
+  the two files live in different modules).
+- Measured (4 vCPU Xeon 2.8 GHz, go1.24.7): raw parse is 3–10% *faster*
+  than cgo single-threaded and 29% faster on the stress query; `Scan` is
+  at parity. Small-query parallel parse runs 6–26% *behind* cgo on this
+  box: the Go side pays GC for every AST node while libpg_query's arena
+  allocations are invisible to Go accounting, and the cgo crossing itself
+  parallelizes fine. Normalize is 1.6–3.6× slower, fingerprint 4.3–6.3×,
+  ParseToJSON ~5× — all three are protobuf-reflection-driven walks, so
+  the standing option of generated per-node emitters is where that time
+  would come back if those entry points ever matter to a consumer. (cgo's
+  1 alloc/op in these tables is just the Go-side result copy; C-side
+  allocations are not observable, so allocs/op is only meaningful within
+  the pure-Go column.)
+- Profiling (alloc_space on the stress parse) found two fixable hotspots,
+  both landed with the benchmarks. The parser's token buffer regrew via
+  append doubling — 72% of all bytes allocated — and is now pre-sized to
+  `len(input)/3` (the stress case's ~3 bytes/token is the dense extreme),
+  cutting stress parse from 92 MB to 39 MB and 185 ms to 146 ms per op.
+  The fingerprint walk re-sorted each node's field descriptors on every
+  visit; the per-type order is static and now memoized (−28–31% time,
+  −25% allocs).
+- What remains is the tree itself: ~2 allocations per AST node (the node
+  struct plus its `ast.Node` oneof wrapper) and the final protobuf
+  marshal — 20–138 allocs for the upstream benchmark queries. That is the
+  cost of the pg_query_go-compatible protobuf AST, not overhead to
+  engineer away.
+- Still open from the milestone: difftest mutation fuzzing as a scheduled
+  CI job, `go test -race` over the parallel corpus run, and the wasilibs
+  (wasm) comparison.
+
 ## Regeneration (the PostgreSQL-upgrade story)
 
 Everything derived is derived by committed tooling from the pin:

benchmark_test.go

@@ -0,0 +1,234 @@
+// Benchmarks mirroring pg_query_go v6.2.2's benchmark_test.go: same
+// benchmark names, same inputs, so `benchstat` can diff a run of this file
+// against a run of oracle/benchmark_test.go (the cgo twin) directly.
+//
+// The Stress benchmarks use the largest corpus input — the 1.1 MB
+// multi-VALUES INSERT (fingerprint suite, case 073) — the same query
+// PLAN.md's milestone 12 names for memory profiling.
+package oliphant_test
+
+import (
+	"testing"
+
+	pg_query "github.com/sqlc-dev/oliphant"
+	"github.com/sqlc-dev/oliphant/internal/testfile"
+	"github.com/sqlc-dev/oliphant/parser"
+)
+
+// Prevent compiler optimizations by assigning all results to global variables
+// (same trick as upstream's benchmark file).
+var (
+	benchErr  error
+	resultStr []byte
+	resultS   string
+	resultRes *pg_query.ParseResult
+)
+
+const stressCaseFile = "parser/testdata/fingerprint/libpg_query.test"
+
+// stressInput returns the 1.1 MB INSERT (case 073 of the fingerprint suite).
+func stressInput(b *testing.B) string {
+	b.Helper()
+	cases, err := testfile.Read(stressCaseFile)
+	if err != nil {
+		b.Fatal(err)
+	}
+	big := 0
+	for i := range cases {
+		if len(cases[i].Input) > len(cases[big].Input) {
+			big = i
+		}
+	}
+	return cases[big].Input
+}
+
+func benchmarkParse(input string, b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		resultRes, benchErr = pg_query.Parse(input)
+		if benchErr != nil {
+			b.Errorf("Benchmark produced error %s\n\n", benchErr)
+		}
+	}
+}
+
+func benchmarkParseParallel(input string, b *testing.B) {
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			_, err := pg_query.Parse(input)
+			if err != nil {
+				b.Errorf("Benchmark produced error %s\n\n", err)
+			}
+		}
+	})
+}
+
+func benchmarkRawParse(input string, b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		resultStr, benchErr = parser.ParseToProtobuf(input)
+		if benchErr != nil {
+			b.Errorf("Benchmark produced error %s\n\n", benchErr)
+		}
+		if len(resultStr) == 0 {
+			b.Errorf("Benchmark produced empty result\n\n")
+		}
+	}
+}
+
+func benchmarkRawParseParallel(input string, b *testing.B) {
+	b.RunParallel(func(pb *testing.PB) {
+		var str []byte
+		var err error
+		for pb.Next() {
+			str, err = parser.ParseToProtobuf(input)
+			if err != nil {
+				b.Errorf("Benchmark produced error %s\n\n", err)
+			}
+			if len(str) == 0 {
+				b.Errorf("Benchmark produced empty result\n\n")
+			}
+		}
+	})
+}
+
+func benchmarkParseToJSON(input string, b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		resultS, benchErr = pg_query.ParseToJSON(input)
+		if benchErr != nil {
+			b.Errorf("Benchmark produced error %s\n\n", benchErr)
+		}
+	}
+}
+
+func benchmarkScan(input string, b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_, benchErr = pg_query.Scan(input)
+		if benchErr != nil {
+			b.Errorf("Benchmark produced error %s\n\n", benchErr)
+		}
+	}
+}
+
+func benchmarkFingerprint(input string, b *testing.B) {
+	var str string
+	for i := 0; i < b.N; i++ {
+		str, benchErr = pg_query.Fingerprint(input)
+		if benchErr != nil {
+			b.Errorf("Benchmark produced error %s\n\n", benchErr)
+		}
+		if str == "" {
+			b.Errorf("Benchmark produced empty result\n\n")
+		}
+	}
+}
+
+func benchmarkNormalize(input string, b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		resultS, benchErr = pg_query.Normalize(input)
+		if benchErr != nil {
+			b.Errorf("Benchmark produced error %s\n\n", benchErr)
+		}
+		if resultS == "" {
+			b.Errorf("Benchmark produced empty result\n\n")
+		}
+	}
+}
+
+func BenchmarkParseSelect1(b *testing.B) {
+	benchmarkParse("SELECT 1", b)
+}
+func BenchmarkParseSelect2(b *testing.B) {
+	benchmarkParse("SELECT 1 FROM x WHERE y IN ('a', 'b', 'c')", b)
+}
+func BenchmarkParseCreateTable(b *testing.B) {
+	benchmarkParse("CREATE TABLE types (a float(2), b float(49), c NUMERIC(2, 3), d character(4), e char(5), f varchar(6), g character varying(7))", b)
+}
+
+func BenchmarkParseSelect1Parallel(b *testing.B) {
+	benchmarkParseParallel("SELECT 1", b)
+}
+func BenchmarkParseSelect2Parallel(b *testing.B) {
+	benchmarkParseParallel("SELECT 1 FROM x WHERE y IN ('a', 'b', 'c')", b)
+}
+func BenchmarkParseCreateTableParallel(b *testing.B) {
+	benchmarkParseParallel("CREATE TABLE types (a float(2), b float(49), c NUMERIC(2, 3), d character(4), e char(5), f varchar(6), g character varying(7))", b)
+}
+
+func BenchmarkRawParseSelect1(b *testing.B) {
+	benchmarkRawParse("SELECT 1", b)
+}
+func BenchmarkRawParseSelect2(b *testing.B) {
+	benchmarkRawParse("SELECT 1 FROM x WHERE y IN ('a', 'b', 'c')", b)
+}
+func BenchmarkRawParseCreateTable(b *testing.B) {
+	benchmarkRawParse("CREATE TABLE types (a float(2), b float(49), c NUMERIC(2, 3), d character(4), e char(5), f varchar(6), g character varying(7))", b)
+}
+
+func BenchmarkRawParseSelect1Parallel(b *testing.B) {
+	benchmarkRawParseParallel("SELECT 1", b)
+}
+func BenchmarkRawParseSelect2Parallel(b *testing.B) {
+	benchmarkRawParseParallel("SELECT 1 FROM x WHERE y IN ('a', 'b', 'c')", b)
+}
+func BenchmarkRawParseCreateTableParallel(b *testing.B) {
+	benchmarkRawParseParallel("CREATE TABLE types (a float(2), b float(49), c NUMERIC(2, 3), d character(4), e char(5), f varchar(6), g character varying(7))", b)
+}
+
+func BenchmarkFingerprintSelect1(b *testing.B) {
+	benchmarkFingerprint("SELECT 1", b)
+}
+func BenchmarkFingerprintSelect2(b *testing.B) {
+	benchmarkFingerprint("SELECT 1 FROM x WHERE y IN ('a', 'b', 'c')", b)
+}
+func BenchmarkFingerprintCreateTable(b *testing.B) {
+	benchmarkFingerprint("CREATE TABLE types (a float(2), b float(49), c NUMERIC(2, 3), d character(4), e char(5), f varchar(6), g character varying(7))", b)
+}
+
+func BenchmarkNormalizeSelect1(b *testing.B) {
+	benchmarkNormalize("SELECT 1", b)
+}
+func BenchmarkNormalizeSelect2(b *testing.B) {
+	benchmarkNormalize("SELECT 1 FROM x WHERE y IN ('a', 'b', 'c')", b)
+}
+func BenchmarkNormalizeCreateTable(b *testing.B) {
+	benchmarkNormalize("CREATE TABLE types (a float(2), b float(49), c NUMERIC(2, 3), d character(4), e char(5), f varchar(6), g character varying(7))", b)
+}
+
+// --- beyond the upstream set ---
+
+func BenchmarkParseToJSONSelect2(b *testing.B) {
+	benchmarkParseToJSON("SELECT 1 FROM x WHERE y IN ('a', 'b', 'c')", b)
+}
+func BenchmarkScanSelect2(b *testing.B) {
+	benchmarkScan("SELECT 1 FROM x WHERE y IN ('a', 'b', 'c')", b)
+}
+
+func BenchmarkRawParseStress(b *testing.B) {
+	input := stressInput(b)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkRawParse(input, b)
+}
+func BenchmarkRawParseStressParallel(b *testing.B) {
+	input := stressInput(b)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkRawParseParallel(input, b)
+}
+func BenchmarkFingerprintStress(b *testing.B) {
+	input := stressInput(b)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkFingerprint(input, b)
+}
+func BenchmarkNormalizeStress(b *testing.B) {
+	input := stressInput(b)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkNormalize(input, b)
+}
+func BenchmarkScanStress(b *testing.B) {
+	input := stressInput(b)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkScan(input, b)
+}

internal/fingerprint/fingerprint.go

@@ -22,13 +22,34 @@ package fingerprint
 import (
 	"sort"
 	"strconv"
+	"sync"
 
 	"google.golang.org/protobuf/reflect/protoreflect"
 
 	"github.com/sqlc-dev/oliphant/ast"
 	"github.com/sqlc-dev/oliphant/internal/xxh3"
 )
 
+// fieldOrderCache memoizes each message type's alphabetical field order
+// (keyed by full name): the order is static per type, and re-sorting on
+// every node visit dominated the fingerprint walk's time and allocations.
+var fieldOrderCache sync.Map // protoreflect.FullName -> []protoreflect.FieldDescriptor
+
+// sortedFields returns d's fields in alphabetical (C generator) order.
+func sortedFields(d protoreflect.MessageDescriptor) []protoreflect.FieldDescriptor {
+	if v, ok := fieldOrderCache.Load(d.FullName()); ok {
+		return v.([]protoreflect.FieldDescriptor)
+	}
+	fds := d.Fields()
+	order := make([]protoreflect.FieldDescriptor, fds.Len())
+	for i := 0; i < fds.Len(); i++ {
+		order[i] = fds.Get(i)
+	}
+	sort.Slice(order, func(a, b int) bool { return order[a].JSONName() < order[b].JSONName() })
+	fieldOrderCache.Store(d.FullName(), order)
+	return order
+}
+
 // fingerprintVersion is PG_QUERY_FINGERPRINT_VERSION, the XXH3 seed.
 const fingerprintVersion = 3
 
@@ -286,11 +307,7 @@ func (ctx *context) fields(m protoreflect.Message, parentType, fieldName string,
 	}
 
 	fds := d.Fields()
-	order := make([]protoreflect.FieldDescriptor, fds.Len())
-	for i := 0; i < fds.Len(); i++ {
-		order[i] = fds.Get(i)
-	}
-	sort.Slice(order, func(a, b int) bool { return order[a].JSONName() < order[b].JSONName() })
+	order := sortedFields(d)
 
 	for _, fd := range order {
 		name := fd.JSONName()

internal/parse/modes.go

@@ -29,7 +29,7 @@ func ParseWithMode(input string, mode Mode) (res *ast.ParseResult, err *lexer.Er
 		return Parse(input)
 	}
 	s := lexer.New(input)
-	p := &parser{src: s.Input(), filter: lexer.NewFilter(s)}
+	p := &parser{src: s.Input(), filter: lexer.NewFilter(s), toks: make([]lexer.Token, 0, tokenCap(input))}
 	defer func() {
 		if r := recover(); r != nil {
 			if b, ok := r.(bail); ok {

internal/parse/parser.go

@@ -32,10 +32,20 @@ type parser struct {
 // returns through several hundred productions would bury the grammar.
 type bail struct{ err *lexer.Error }
 
+// tokenCap sizes the token buffer up front: SQL averages a handful of bytes
+// per token, and the dense extreme (large VALUES parameter lists, the corpus
+// stress case) runs ~3 bytes/token, so len/3 keeps even pathological inputs
+// from regrowing the buffer — append-regrowth otherwise dominates
+// alloc_space on large statements — while short queries over-allocate at
+// most a few hundred transient bytes.
+func tokenCap(input string) int {
+	return len(input)/3 + 8
+}
+
 // Parse is raw_parser for RAW_PARSE_DEFAULT: parse_toplevel/stmtmulti.
 func Parse(input string) (res *ast.ParseResult, err *lexer.Error) {
 	s := lexer.New(input)
-	p := &parser{src: s.Input(), filter: lexer.NewFilter(s)}
+	p := &parser{src: s.Input(), filter: lexer.NewFilter(s), toks: make([]lexer.Token, 0, tokenCap(input))}
 	defer func() {
 		if r := recover(); r != nil {
 			if b, ok := r.(bail); ok {