From bfbf829cd6415e17ff89c868c23b6509a3687570 Mon Sep 17 00:00:00 2001 From: N Rohith Reddy Date: Sat, 18 Jul 2026 21:35:09 -0400 Subject: [PATCH 1/2] test: reproduce PreferredModels semantic-cache collision (#789) Signed-off-by: N Rohith Reddy Co-authored-by: Cursor --- internal/proxy/service_cache_test.go | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/internal/proxy/service_cache_test.go b/internal/proxy/service_cache_test.go index f57fe0520..64dc6f5b1 100644 --- a/internal/proxy/service_cache_test.go +++ b/internal/proxy/service_cache_test.go @@ -237,3 +237,40 @@ func TestService_Cache_DisabledByNilCache(t *testing.T) { assert.Len(t, provider.proxyBodies, 2, "nil cache must be a transparent passthrough") assert.Empty(t, rec2.Header().Get(proxy.HeaderRouterCache)) } + +// TestService_Cache_PreferredModelsBypasses guards #789: PreferredModels is a +// per-request score-perturbing input (blendScoresV2 priorityBonus) that can +// flip the scorer's winner without changing EffectiveKnobsHash. Without a +// cacheEligible gate, a no-preference store is replayed to a preference- +// bearing request (wrong model's body, live x-router-model). Same class as +// the subsidyFactors gate from #495. +func TestService_Cache_PreferredModelsBypasses(t *testing.T) { + emb := embeddingFixture(8) + provider := &fakeProvider{ + proxyResponse: func(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"from-no-pref","content":"opus-shaped"}`)) + }, + } + // Fake router returns a fixed decision+embedding so both turns land in the + // same cache bucket — isolating the eligibility gate from scorer variance. + fr := &fakeRouter{decision: decisionWithEmbedding(emb, []int{0, 1, 2, 3})} + c := cache.New(cache.DefaultConfig()) + svc := proxy.NewService(fr, map[string]providers.Client{providers.ProviderAnthropic: provider}, nil, false, c, nil, false, providers.ProviderAnthropic, "claude-haiku-4-5", nil) + + body := anthropicBody("close-call prompt", false) + + ctxNoPref := proxyContextWithExternalID(t, "tenant-pref") + rec1 := httptest.NewRecorder() + require.NoError(t, svc.ProxyMessages(ctxNoPref, body, rec1, httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader("")))) + require.Len(t, provider.proxyBodies, 1, "no-preference turn must miss and hit the provider") + + ctxPref := context.WithValue(ctxNoPref, proxy.InstallationPreferredModelsContextKey{}, []string{"claude-haiku-4-5"}) + rec2 := httptest.NewRecorder() + require.NoError(t, svc.ProxyMessages(ctxPref, body, rec2, httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader("")))) + + assert.Len(t, provider.proxyBodies, 2, + "PreferredModels turn must bypass the semantic cache and hit the provider (no cross-preference hit)") + assert.Empty(t, rec2.Header().Get(proxy.HeaderRouterCache), + "PreferredModels turn must not report x-router-cache: hit") +} From dec570660e20f92858b00b63855cd3d11987e4f0 Mon Sep 17 00:00:00 2001 From: N Rohith Reddy Date: Sat, 18 Jul 2026 21:35:40 -0400 Subject: [PATCH 2/2] fix(router): exclude PreferredModels turns from semantic cache (#789) Mirrors the existing subsidyFactors/SubscriptionOnly gates on cacheEligible: PreferredModels is a per-request score-perturbing input (blendScoresV2 adds priorityBonus additively, same as subsidy) that can change the scorer's winning model without changing EffectiveKnobsHash, so a stored response can be served across a preference-driven winner change. Gating matches #495/#747 precedent rather than folding into the hash (#214/#338's mixModel pattern), since PreferredModels is structurally a score-perturbing input like subsidy, not a post-hoc override like bandit exploration. Fixes #789 Signed-off-by: N Rohith Reddy Co-authored-by: Cursor --- internal/proxy/service.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/internal/proxy/service.go b/internal/proxy/service.go index 623699481..f14875b8e 100644 --- a/internal/proxy/service.go +++ b/internal/proxy/service.go @@ -2283,12 +2283,13 @@ func (s *Service) ProxyMessages(ctx context.Context, body []byte, w http.Respons // Semantic-cache eligibility: configured, non-streaming, decision has // metadata, externalID present, not eval traffic. Skip when a compaction // handover rewrote env (embedding predates the rewrite) or when subsidy - // factors are non-empty (the cache key doesn't capture quota-headroom- - // dependent model choice; subsidyFactors returns nil when the feature is off). - // Subscription-only turns are excluded (like the OpenAI path): the mode is an - // unfoldable routing signal absent from the cache key, so a stored body would - // bypass the exhausted-sub 402 guard and the depleted-credits warning below. - cacheEligible := s.semanticCache != nil && !env.Stream() && decision.Metadata != nil && externalID != "" && !bypassEval && !compactionHandoverRan && !billing.SubscriptionOnlyFromContext(ctx) && len(s.subsidyFactors(ctx, r.Header)) == 0 + // factors / PreferredModels are non-empty (the cache key doesn't capture + // those score-perturbing inputs' effect on model choice; subsidyFactors + // returns nil when the feature is off). Subscription-only turns are + // excluded (like the OpenAI path): the mode is an unfoldable routing signal + // absent from the cache key, so a stored body would bypass the exhausted-sub + // 402 guard and the depleted-credits warning below. + cacheEligible := s.semanticCache != nil && !env.Stream() && decision.Metadata != nil && externalID != "" && !bypassEval && !compactionHandoverRan && !billing.SubscriptionOnlyFromContext(ctx) && len(s.subsidyFactors(ctx, r.Header)) == 0 && len(s.preferredModelsForRequest(ctx)) == 0 if cacheEligible { if resp, hit := s.semanticCache.Lookup(externalID, cache.FormatAnthropic, decision.Metadata.Embedding, decision.Metadata.ClusterIDs, decision.Metadata.ClusterRouterVersion, decision.Metadata.EffectiveKnobsHash); hit { s.writeCachedResponse(w, resp, decision) @@ -4213,9 +4214,10 @@ func (s *Service) ProxyOpenAIChatCompletion(ctx context.Context, body []byte, w pinAgeSec := routeRes.PinAgeSec s.logPlannerOutcome(ctx, routeRes) - // See the ProxyMessages cache-eligibility note: subsidized requests bypass the - // semantic cache (the key doesn't capture headroom-dependent model choice). - cacheEligible := s.semanticCache != nil && !env.Stream() && decision.Metadata != nil && externalID != "" && !bypassEval && !responsesPassthrough && !billing.SubscriptionOnlyFromContext(ctx) && len(s.subsidyFactors(ctx, r.Header)) == 0 + // See the ProxyMessages cache-eligibility note: subsidized and + // PreferredModels requests bypass the semantic cache (the key doesn't + // capture those score-perturbing inputs' effect on model choice). + cacheEligible := s.semanticCache != nil && !env.Stream() && decision.Metadata != nil && externalID != "" && !bypassEval && !responsesPassthrough && !billing.SubscriptionOnlyFromContext(ctx) && len(s.subsidyFactors(ctx, r.Header)) == 0 && len(s.preferredModelsForRequest(ctx)) == 0 if cacheEligible { if resp, hit := s.semanticCache.Lookup(externalID, cache.FormatOpenAI, decision.Metadata.Embedding, decision.Metadata.ClusterIDs, decision.Metadata.ClusterRouterVersion, decision.Metadata.EffectiveKnobsHash); hit { s.writeCachedResponse(w, resp, decision)