Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 109 additions & 4 deletions internal/router/cluster/distribution_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,20 @@ func TestRoutingDistribution_DefaultGridAndV1Guard(t *testing.T) {
func TestRoutingDistribution_NoDeadZone(t *testing.T) {
// Regression guard: a run of identical mixes across adjacent dial steps is
// what made "50% look like 20%" — the calibration must keep steps live.
s := loadV0_67(t)
// Floored bundles: TestRoutingDistribution_NoDeadZone_FlooredBundle.
assertNoDeadZone(t, loadV0_67(t))
}

func TestRoutingDistribution_NoDeadZone_FlooredBundle(t *testing.T) {
// #779: unfloored calibration left a long consecutive dead zone on v0.73.
points, err := loadV0_73(t).RoutingDistribution(21, nil, nil)
require.NoError(t, err)
assert.LessOrEqual(t, maxIdenticalMixRun(points), 2,
"floored bundle: consecutive identical-mix run too long — dial dead zone")
}

func assertNoDeadZone(t *testing.T, s *Scorer) {
t.Helper()
points, err := s.RoutingDistribution(21, nil, nil)
require.NoError(t, err)

Expand All @@ -144,10 +157,35 @@ func TestRoutingDistribution_NoDeadZone(t *testing.T) {
"too many adjacent dial positions route an identical mix (%d) — dial has a dead zone", identicalRuns)
}

// maxIdenticalMixRun is the longest streak of identical adjacent mixes.
func maxIdenticalMixRun(points []DistributionPoint) int {
maxRun, cur := 0, 0
for i := 1; i < len(points); i++ {
if mixSignatureOf(points[i].Models) == mixSignatureOf(points[i-1].Models) {
cur++
if cur > maxRun {
maxRun = cur
}
} else {
cur = 0
}
}
return maxRun
}

func TestRoutingDistribution_MidDialIsPricierThanLowDial(t *testing.T) {
// Reported bug: mid dial (0.5) used to route the same all-cheapest mix as
// low dial (0.2); it must now route a meaningfully pricier mix.
s := loadV0_67(t)
assertMidDialPricierThanLow(t, loadV0_67(t), 1.5)
}

func TestRoutingDistribution_MidDialIsPricierThanLowDial_FlooredBundle(t *testing.T) {
// #779: pre-fix cost(0.5)==cost(0.2); 1.2× (not v0.67's 1.5×) fits floored span.
assertMidDialPricierThanLow(t, loadV0_73(t), 1.2)
}

func assertMidDialPricierThanLow(t *testing.T, s *Scorer, minRatio float64) {
t.Helper()
points, err := s.RoutingDistribution(21, nil, nil)
require.NoError(t, err)

Expand All @@ -162,8 +200,8 @@ func TestRoutingDistribution_MidDialIsPricierThanLowDial(t *testing.T) {
}
require.NotEmpty(t, low.Models, "dial position 0.2 must be present in the 21-point grid")
require.NotEmpty(t, mid.Models)
assert.Greater(t, mid.ProjectedCostPer1KInputUSD, low.ProjectedCostPer1KInputUSD*1.5,
"the 50%% dial must route a meaningfully pricier (higher-quality) mix than the 20%% dial")
assert.Greater(t, mid.ProjectedCostPer1KInputUSD, low.ProjectedCostPer1KInputUSD*minRatio,
"the 50%% dial must route a meaningfully pricier (higher-quality) mix than the 20%% dial (ratio > %.2f)", minRatio)
}

// loadV0_70 loads the committed v0.70 bundle (default alpha protects
Expand All @@ -178,6 +216,73 @@ func loadV0_70(t *testing.T) *Scorer {
return s
}

// loadV0_73 loads the floored bundle used to pin #779.
func loadV0_73(t *testing.T) *Scorer {
t.Helper()
bundle, err := LoadBundle("v0.73")
require.NoError(t, err)
require.True(t, bundle.IsV2)
require.NotEmpty(t, bundle.Metadata.Training.DefaultRoutingKnobs.AlphaFloor,
"v0.73 must ship alpha_floor — otherwise this is not a floored-bundle fixture")
s, err := NewScorer(bundle, DefaultConfig(), &fakeEmbedder{dim: bundle.Centroids.Dim}, allProviders())
require.NoError(t, err)
return s
}

func TestApplyAlphaFloor_NilAndHeterogeneous(t *testing.T) {
alpha := []float64{0, 0, 0}
applyAlphaFloor(alpha, 0.2, nil)
assert.Equal(t, []float64{0.2, 0.2, 0.2}, alpha, "nil floor writes raw into every slot")

floor := []float64{0.4, 0.1, 0.7}
applyAlphaFloor(alpha, 0.3, floor)
assert.InDelta(t, 0.4, alpha[0], 1e-9, "slot above raw is held at floor")
assert.InDelta(t, 0.3, alpha[1], 1e-9, "slot below raw follows raw")
assert.InDelta(t, 0.7, alpha[2], 1e-9, "higher floor still wins")
}

func TestComputeDialCalibration_NoBreakpointsBelowMinFloor(t *testing.T) {
// #779: no interior breakpoint below min(AlphaFloor) — those alphas are unreachable.
s := loadV0_73(t)
floor := s.defaultActiveKnobs().AlphaFloor
require.NotEmpty(t, floor)
minFloor := floor[0]
for _, f := range floor {
if f < minFloor {
minFloor = f
}
}
for _, a := range s.dialAlphaBreakpoints {
if a == 0 || a == 1 {
continue
}
assert.GreaterOrEqual(t, a, minFloor,
"breakpoint alpha=%.4f is below min_floor=%.2f — calibration swept an unreachable region", a, minFloor)
}
}

func TestComputeDialCalibration_HeterogeneousFloorSynthetic(t *testing.T) {
// Heterogeneous floors: no breakpoint in (0, minFloor).
const lo, hi = 0.45, 0.70
s := newV2BundleForTest(t, &fakeEmbedder{vec: makeOpusVec()}, v2BundleOpts{
defaultKnobs: &DefaultRoutingKnobs{
Alpha: []float64{0.5, 0.5},
AlphaFloor: []float64{lo, hi},
SpeedWeight: 0.0,
OutputCostRatio: 1.0, // cost axis active so alpha travel moves the mix
ExpectedOutputTokens: 2000,
},
})
require.GreaterOrEqual(t, len(s.dialAlphaBreakpoints), 2)
for _, a := range s.dialAlphaBreakpoints {
if a == 0 || a == 1 {
continue
}
assert.GreaterOrEqual(t, a, lo,
"heterogeneous-floor synthetic: breakpoint %.4f below min floor %.2f", a, lo)
}
}

func TestApplyDialAlpha_HoldsEachClusterAtItsDeclaredFloor(t *testing.T) {
s := loadV0_70(t)
knobs := s.defaultActiveKnobs()
Expand Down
31 changes: 18 additions & 13 deletions internal/router/cluster/scorer.go
Original file line number Diff line number Diff line change
Expand Up @@ -242,22 +242,23 @@ const qualityBiasCalibrationGrid = 401
// dialToAlpha interpolates across the breakpoints instead, so equal dial
// travel crosses an equal number of mix changes.
//
// The sweep applies AlphaFloor via applyAlphaFloor (same as applyDialAlpha)
// so breakpoints exclude unreachable low-alpha mix changes (#779).
// Returns nil when fewer than two distinct mixes exist, so dialToAlpha falls
// back to the identity.
func (s *Scorer) computeDialCalibration() []float64 {
k := s.centroids.K
centroidTopClusters := s.allCentroidTopClusters()

base := s.defaultActiveKnobs()
floor := base.AlphaFloor
breakpoints := make([]float64, 0, 32)
prevSig := ""
for g := 0; g < qualityBiasCalibrationGrid; g++ {
a := float64(g) / float64(qualityBiasCalibrationGrid-1)
knobs := base
knobs.Alpha = make([]float64, k)
for i := range knobs.Alpha {
knobs.Alpha[i] = a
}
applyAlphaFloor(knobs.Alpha, a, floor)
counts := make(map[string]int, len(s.models))
for c := 0; c < k; c++ {
scores := s.blendScoresV2(centroidTopClusters[c], knobs, s.models, nil, nil)
Expand Down Expand Up @@ -322,22 +323,26 @@ func (s *Scorer) dialToAlpha(t float64) float64 {
return bp[i] + frac*(bp[i+1]-bp[i])
}

// applyAlphaFloor writes alpha[i] = max(raw, floor[i]). floor==nil disables
// flooring. Shared by computeDialCalibration and applyDialAlpha (#779).
func applyAlphaFloor(alpha []float64, raw float64, floor []float64) {
for i := range alpha {
if floor != nil && floor[i] > raw {
alpha[i] = floor[i]
} else {
alpha[i] = raw
}
}
}

// applyDialAlpha resolves dial position t to per-cluster alpha in place:
// alpha[i] = max(dialToAlpha(t), floor[i]). floor is the lowest quality
// weight the bundle tolerates per cluster at max price-sensitivity, so a
// price-leaning dial can't collapse the whole vector onto the cheapest model
// (which stranded agentic turns on models that can't drive the harness).
// floor==nil disables flooring. Single source of truth shared by Route and
// RoutingDistribution; caller guarantees len(floor)==len(alpha) when non-nil.
// floor==nil disables flooring. Shared by Route and RoutingDistribution.
func (s *Scorer) applyDialAlpha(t float64, alpha, floor []float64) {
a := s.dialToAlpha(t)
for i := range alpha {
if floor != nil && floor[i] > a {
alpha[i] = floor[i]
} else {
alpha[i] = a
}
}
applyAlphaFloor(alpha, s.dialToAlpha(t), floor)
}

// resolveProviderFor walks the catalog's ordered ProviderBinding list for
Expand Down