diff --git a/internal/router/cluster/distribution_test.go b/internal/router/cluster/distribution_test.go index 9257fa935..c4cb2f266 100644 --- a/internal/router/cluster/distribution_test.go +++ b/internal/router/cluster/distribution_test.go @@ -128,7 +128,20 @@ func TestRoutingDistribution_DefaultGridAndV1Guard(t *testing.T) { func TestRoutingDistribution_NoDeadZone(t *testing.T) { // Regression guard: a run of identical mixes across adjacent dial steps is // what made "50% look like 20%" — the calibration must keep steps live. - s := loadV0_67(t) + // Floored bundles: TestRoutingDistribution_NoDeadZone_FlooredBundle. + assertNoDeadZone(t, loadV0_67(t)) +} + +func TestRoutingDistribution_NoDeadZone_FlooredBundle(t *testing.T) { + // #779: unfloored calibration left a long consecutive dead zone on v0.73. + points, err := loadV0_73(t).RoutingDistribution(21, nil, nil) + require.NoError(t, err) + assert.LessOrEqual(t, maxIdenticalMixRun(points), 2, + "floored bundle: consecutive identical-mix run too long — dial dead zone") +} + +func assertNoDeadZone(t *testing.T, s *Scorer) { + t.Helper() points, err := s.RoutingDistribution(21, nil, nil) require.NoError(t, err) @@ -144,10 +157,35 @@ func TestRoutingDistribution_NoDeadZone(t *testing.T) { "too many adjacent dial positions route an identical mix (%d) — dial has a dead zone", identicalRuns) } +// maxIdenticalMixRun is the longest streak of identical adjacent mixes. +func maxIdenticalMixRun(points []DistributionPoint) int { + maxRun, cur := 0, 0 + for i := 1; i < len(points); i++ { + if mixSignatureOf(points[i].Models) == mixSignatureOf(points[i-1].Models) { + cur++ + if cur > maxRun { + maxRun = cur + } + } else { + cur = 0 + } + } + return maxRun +} + func TestRoutingDistribution_MidDialIsPricierThanLowDial(t *testing.T) { // Reported bug: mid dial (0.5) used to route the same all-cheapest mix as // low dial (0.2); it must now route a meaningfully pricier mix. - s := loadV0_67(t) + assertMidDialPricierThanLow(t, loadV0_67(t), 1.5) +} + +func TestRoutingDistribution_MidDialIsPricierThanLowDial_FlooredBundle(t *testing.T) { + // #779: pre-fix cost(0.5)==cost(0.2); 1.2× (not v0.67's 1.5×) fits floored span. + assertMidDialPricierThanLow(t, loadV0_73(t), 1.2) +} + +func assertMidDialPricierThanLow(t *testing.T, s *Scorer, minRatio float64) { + t.Helper() points, err := s.RoutingDistribution(21, nil, nil) require.NoError(t, err) @@ -162,8 +200,8 @@ func TestRoutingDistribution_MidDialIsPricierThanLowDial(t *testing.T) { } require.NotEmpty(t, low.Models, "dial position 0.2 must be present in the 21-point grid") require.NotEmpty(t, mid.Models) - assert.Greater(t, mid.ProjectedCostPer1KInputUSD, low.ProjectedCostPer1KInputUSD*1.5, - "the 50%% dial must route a meaningfully pricier (higher-quality) mix than the 20%% dial") + assert.Greater(t, mid.ProjectedCostPer1KInputUSD, low.ProjectedCostPer1KInputUSD*minRatio, + "the 50%% dial must route a meaningfully pricier (higher-quality) mix than the 20%% dial (ratio > %.2f)", minRatio) } // loadV0_70 loads the committed v0.70 bundle (default alpha protects @@ -178,6 +216,73 @@ func loadV0_70(t *testing.T) *Scorer { return s } +// loadV0_73 loads the floored bundle used to pin #779. +func loadV0_73(t *testing.T) *Scorer { + t.Helper() + bundle, err := LoadBundle("v0.73") + require.NoError(t, err) + require.True(t, bundle.IsV2) + require.NotEmpty(t, bundle.Metadata.Training.DefaultRoutingKnobs.AlphaFloor, + "v0.73 must ship alpha_floor — otherwise this is not a floored-bundle fixture") + s, err := NewScorer(bundle, DefaultConfig(), &fakeEmbedder{dim: bundle.Centroids.Dim}, allProviders()) + require.NoError(t, err) + return s +} + +func TestApplyAlphaFloor_NilAndHeterogeneous(t *testing.T) { + alpha := []float64{0, 0, 0} + applyAlphaFloor(alpha, 0.2, nil) + assert.Equal(t, []float64{0.2, 0.2, 0.2}, alpha, "nil floor writes raw into every slot") + + floor := []float64{0.4, 0.1, 0.7} + applyAlphaFloor(alpha, 0.3, floor) + assert.InDelta(t, 0.4, alpha[0], 1e-9, "slot above raw is held at floor") + assert.InDelta(t, 0.3, alpha[1], 1e-9, "slot below raw follows raw") + assert.InDelta(t, 0.7, alpha[2], 1e-9, "higher floor still wins") +} + +func TestComputeDialCalibration_NoBreakpointsBelowMinFloor(t *testing.T) { + // #779: no interior breakpoint below min(AlphaFloor) — those alphas are unreachable. + s := loadV0_73(t) + floor := s.defaultActiveKnobs().AlphaFloor + require.NotEmpty(t, floor) + minFloor := floor[0] + for _, f := range floor { + if f < minFloor { + minFloor = f + } + } + for _, a := range s.dialAlphaBreakpoints { + if a == 0 || a == 1 { + continue + } + assert.GreaterOrEqual(t, a, minFloor, + "breakpoint alpha=%.4f is below min_floor=%.2f — calibration swept an unreachable region", a, minFloor) + } +} + +func TestComputeDialCalibration_HeterogeneousFloorSynthetic(t *testing.T) { + // Heterogeneous floors: no breakpoint in (0, minFloor). + const lo, hi = 0.45, 0.70 + s := newV2BundleForTest(t, &fakeEmbedder{vec: makeOpusVec()}, v2BundleOpts{ + defaultKnobs: &DefaultRoutingKnobs{ + Alpha: []float64{0.5, 0.5}, + AlphaFloor: []float64{lo, hi}, + SpeedWeight: 0.0, + OutputCostRatio: 1.0, // cost axis active so alpha travel moves the mix + ExpectedOutputTokens: 2000, + }, + }) + require.GreaterOrEqual(t, len(s.dialAlphaBreakpoints), 2) + for _, a := range s.dialAlphaBreakpoints { + if a == 0 || a == 1 { + continue + } + assert.GreaterOrEqual(t, a, lo, + "heterogeneous-floor synthetic: breakpoint %.4f below min floor %.2f", a, lo) + } +} + func TestApplyDialAlpha_HoldsEachClusterAtItsDeclaredFloor(t *testing.T) { s := loadV0_70(t) knobs := s.defaultActiveKnobs() diff --git a/internal/router/cluster/scorer.go b/internal/router/cluster/scorer.go index f50cbcf72..ebf302bd2 100644 --- a/internal/router/cluster/scorer.go +++ b/internal/router/cluster/scorer.go @@ -242,6 +242,8 @@ const qualityBiasCalibrationGrid = 401 // dialToAlpha interpolates across the breakpoints instead, so equal dial // travel crosses an equal number of mix changes. // +// The sweep applies AlphaFloor via applyAlphaFloor (same as applyDialAlpha) +// so breakpoints exclude unreachable low-alpha mix changes (#779). // Returns nil when fewer than two distinct mixes exist, so dialToAlpha falls // back to the identity. func (s *Scorer) computeDialCalibration() []float64 { @@ -249,15 +251,14 @@ func (s *Scorer) computeDialCalibration() []float64 { centroidTopClusters := s.allCentroidTopClusters() base := s.defaultActiveKnobs() + floor := base.AlphaFloor breakpoints := make([]float64, 0, 32) prevSig := "" for g := 0; g < qualityBiasCalibrationGrid; g++ { a := float64(g) / float64(qualityBiasCalibrationGrid-1) knobs := base knobs.Alpha = make([]float64, k) - for i := range knobs.Alpha { - knobs.Alpha[i] = a - } + applyAlphaFloor(knobs.Alpha, a, floor) counts := make(map[string]int, len(s.models)) for c := 0; c < k; c++ { scores := s.blendScoresV2(centroidTopClusters[c], knobs, s.models, nil, nil) @@ -322,22 +323,26 @@ func (s *Scorer) dialToAlpha(t float64) float64 { return bp[i] + frac*(bp[i+1]-bp[i]) } +// applyAlphaFloor writes alpha[i] = max(raw, floor[i]). floor==nil disables +// flooring. Shared by computeDialCalibration and applyDialAlpha (#779). +func applyAlphaFloor(alpha []float64, raw float64, floor []float64) { + for i := range alpha { + if floor != nil && floor[i] > raw { + alpha[i] = floor[i] + } else { + alpha[i] = raw + } + } +} + // applyDialAlpha resolves dial position t to per-cluster alpha in place: // alpha[i] = max(dialToAlpha(t), floor[i]). floor is the lowest quality // weight the bundle tolerates per cluster at max price-sensitivity, so a // price-leaning dial can't collapse the whole vector onto the cheapest model // (which stranded agentic turns on models that can't drive the harness). -// floor==nil disables flooring. Single source of truth shared by Route and -// RoutingDistribution; caller guarantees len(floor)==len(alpha) when non-nil. +// floor==nil disables flooring. Shared by Route and RoutingDistribution. func (s *Scorer) applyDialAlpha(t float64, alpha, floor []float64) { - a := s.dialToAlpha(t) - for i := range alpha { - if floor != nil && floor[i] > a { - alpha[i] = floor[i] - } else { - alpha[i] = a - } - } + applyAlphaFloor(alpha, s.dialToAlpha(t), floor) } // resolveProviderFor walks the catalog's ordered ProviderBinding list for