From c8f1bb61d15e42ca991fcb88e0585f4c40709092 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 13:34:24 -0400 Subject: [PATCH 01/89] feat: add versioned session binding compatibility layer --- package.json | 1 + src/app/v1/_lib/proxy/forwarder.ts | 14 +- src/app/v1/_lib/proxy/provider-selector.ts | 18 +- src/app/v1/_lib/proxy/response-handler.ts | 9 +- src/lib/redis/client.ts | 15 + src/lib/redis/lua-scripts.ts | 373 +++++++ src/lib/redis/session-binding.ts | 989 ++++++++++++++++++ src/lib/session-manager.ts | 820 +++++++++++---- src/lib/session-tracker.ts | 26 +- tests/configs/integration.config.ts | 1 + tests/configs/session-binding.config.ts | 9 + .../session-binding-versioning-redis.test.ts | 503 +++++++++ tests/unit/lib/redis/client.test.ts | 10 + tests/unit/lib/redis/session-binding.test.ts | 907 ++++++++++++++++ .../lib/session-manager-binding-smart.test.ts | 158 ++- ...anager-terminate-provider-sessions.test.ts | 5 +- .../session-manager-terminate-session.test.ts | 90 +- .../session-manager-versioned-binding.test.ts | 151 +++ .../unit/lib/session-tracker-cleanup.test.ts | 16 +- ...provider-selector-cross-type-model.test.ts | 6 +- ...er-selector-model-mismatch-binding.test.ts | 12 +- .../proxy-forwarder-hedge-first-byte.test.ts | 14 +- ...handler-endpoint-circuit-isolation.test.ts | 13 +- 23 files changed, 3860 insertions(+), 300 deletions(-) create mode 100644 src/lib/redis/session-binding.ts create mode 100644 tests/configs/session-binding.config.ts create mode 100644 tests/integration/session-binding-versioning-redis.test.ts create mode 100644 tests/unit/lib/redis/session-binding.test.ts create mode 100644 tests/unit/lib/session-manager-versioned-binding.test.ts diff --git a/package.json b/package.json index da5ce414b..bd5da0a91 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "test:coverage:proxy-guard-pipeline": "vitest run --config tests/configs/proxy-guard-pipeline.config.ts --coverage", "test:coverage:include-session-id-in-errors": "vitest run --config tests/configs/include-session-id-in-errors.config.ts --coverage", "test:coverage:usage-logs-sessionid-search": "vitest run --config tests/configs/usage-logs-sessionid-search.config.ts --coverage", + "test:coverage:session-binding": "vitest run --config tests/configs/session-binding.config.ts --coverage", "test:ci": "vitest run --reporter=default --reporter=junit --outputFile.junit=reports/vitest-junit.xml", "test:v1": "vitest run --config tests/configs/v1.config.ts --coverage --reporter=verbose && bun scripts/check-v1-critical-coverage.ts", "openapi:generate": "bun scripts/generate-v1-types.ts", diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index cb5dd36f8..632807ef2 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -4554,7 +4554,10 @@ export class ProxyForwarder { abortAllAttempts(attempt, "hedge_loser"); - if (session.sessionId) { + // A non-hedged request is finalized through response-handler. Updating + // here as well would perform a duplicate binding read/CAS before the + // stream has passed its final validation. + if (session.sessionId && isActualHedgeWin) { void (async () => { const bindingResult = await SessionManager.updateSessionBindingSmart( session.sessionId!, @@ -4994,16 +4997,17 @@ export class ProxyForwarder { expectedProviderId: number | null ): Promise { if (!session.sessionId) return; - await SessionManager.clearSessionProvider(session.sessionId, expectedProviderId); + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + await SessionManager.clearSessionProvider(session.sessionId, expectedProviderId, keyId); } private static async clearSessionProviderBindings( session: ProxySession, expectedProviderIds: Iterable ): Promise { - for (const providerId of new Set(expectedProviderIds)) { - await ProxyForwarder.clearSessionProviderBinding(session, providerId); - } + if (!session.sessionId) return; + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + await SessionManager.clearSessionProviders(session.sessionId, expectedProviderIds, keyId); } private static markProviderFailed( diff --git a/src/app/v1/_lib/proxy/provider-selector.ts b/src/app/v1/_lib/proxy/provider-selector.ts index 4dc42b9f0..10c397000 100644 --- a/src/app/v1/_lib/proxy/provider-selector.ts +++ b/src/app/v1/_lib/proxy/provider-selector.ts @@ -466,10 +466,8 @@ export class ProxyProviderResolver { } // 从 Redis 读取该 session 绑定的 provider - const providerId = await SessionManager.getSessionProvider( - session.sessionId, - session.authState?.key?.id ?? null - ); + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + const providerId = await SessionManager.getSessionProvider(session.sessionId, keyId); if (!providerId) { logger.debug("ProviderSelector: Session has no bound provider", { sessionId: session.sessionId, @@ -484,7 +482,7 @@ export class ProxyProviderResolver { sessionId: session.sessionId, providerId, }); - await SessionManager.clearSessionProvider(session.sessionId, providerId); + await SessionManager.clearSessionProvider(session.sessionId, providerId, keyId); return null; } @@ -494,7 +492,7 @@ export class ProxyProviderResolver { providerId: provider.id, providerName: provider.name, }); - await SessionManager.clearSessionProvider(session.sessionId, providerId); + await SessionManager.clearSessionProvider(session.sessionId, providerId, keyId); return null; } @@ -508,7 +506,7 @@ export class ProxyProviderResolver { activeTimeEnd: provider.activeTimeEnd, timezone: systemTimezone, }); - await SessionManager.clearSessionProvider(session.sessionId, providerId); + await SessionManager.clearSessionProvider(session.sessionId, providerId, keyId); return null; } @@ -549,7 +547,7 @@ export class ProxyProviderResolver { providerType: provider.providerType, originalFormat: session.originalFormat, }); - await SessionManager.clearSessionProvider(session.sessionId, providerId); + await SessionManager.clearSessionProvider(session.sessionId, providerId, keyId); return null; } @@ -568,7 +566,7 @@ export class ProxyProviderResolver { // 清除过时绑定，避免 SET NX 死锁 // 当 session 内请求模型发生变化时，旧绑定已无意义， // 清除后新的成功请求可通过 SET NX 重新绑定匹配的 provider - await SessionManager.clearSessionProvider(session.sessionId, providerId); + await SessionManager.clearSessionProvider(session.sessionId, providerId, keyId); logger.info("ProviderSelector: Cleared stale provider binding (model mismatch)", { sessionId: session.sessionId, staleProviderId: provider.id, @@ -624,7 +622,7 @@ export class ProxyProviderResolver { ], }, }); - await SessionManager.clearSessionProvider(session.sessionId, providerId); + await SessionManager.clearSessionProvider(session.sessionId, providerId, keyId); return null; } diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index d633c8cbd..acd5eb26f 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -1149,7 +1149,8 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const providerIdForPersistence = meta?.providerId ?? provider?.id ?? null; const clearSessionBinding = async () => { if (!session.sessionId) return; - await SessionManager.clearSessionProvider(session.sessionId, providerIdForPersistence); + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + await SessionManager.clearSessionProvider(session.sessionId, providerIdForPersistence, keyId); }; const isHedgeWinner = meta?.isHedgeWinner === true; @@ -1806,7 +1807,8 @@ export class ProxyResponseHandler { if (session.sessionId) { const sessionId = session.sessionId; postTerminalSideEffects.push(async () => { - await SessionManager.clearSessionProvider(sessionId, provider.id); + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + await SessionManager.clearSessionProvider(sessionId, provider.id, keyId); }); } if ( @@ -1976,7 +1978,8 @@ export class ProxyResponseHandler { if (session.sessionId) { const sessionId = session.sessionId; postTerminalSideEffects.push(async () => { - await SessionManager.clearSessionProvider(sessionId, provider.id); + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + await SessionManager.clearSessionProvider(sessionId, provider.id, keyId); const sessionUsagePayload: SessionUsageUpdate = { status: diff --git a/src/lib/redis/client.ts b/src/lib/redis/client.ts index 21bdc38dc..ae827d95e 100644 --- a/src/lib/redis/client.ts +++ b/src/lib/redis/client.ts @@ -3,6 +3,7 @@ import { getEnvConfig } from "@/lib/config/env.schema"; import { logger } from "@/lib/logger"; let redisClient: Redis | null = null; +let redisClientUrl: string | null = null; function maskRedisUrl(redisUrl: string) { try { @@ -105,9 +106,20 @@ export function getRedisClient(input?: { allowWhenRateLimitDisabled?: boolean }) const safeRedisUrl = maskRedisUrl(redisUrl); + if (redisClient && redisClientUrl !== redisUrl) { + const staleClient = redisClient; + redisClient = null; + redisClientUrl = null; + staleClient.disconnect(); + logger.warn("[Redis] Connection configuration changed, replacing client", { + redisUrl: safeRedisUrl, + }); + } + if (redisClient) { if (redisClient.status === "end") { redisClient = null; + redisClientUrl = null; } else { return redisClient; } @@ -123,6 +135,7 @@ export function getRedisClient(input?: { allowWhenRateLimitDisabled?: boolean }) // 3. 使用组合后的配置创建客户端 const client = new Redis(redisUrl, redisOptions); redisClient = client; + redisClientUrl = redisUrl; // 4. 保持原始的事件监听器 client.on("connect", () => { @@ -153,6 +166,7 @@ export function getRedisClient(input?: { allowWhenRateLimitDisabled?: boolean }) if (redisClient !== client) return; logger.warn("[Redis] Connection ended, resetting client", { redisUrl: safeRedisUrl }); redisClient = null; + redisClientUrl = null; }); // 5. 返回客户端实例 @@ -179,6 +193,7 @@ export async function closeRedis(): Promise { } finally { if (redisClient === client) { redisClient = null; + redisClientUrl = null; } } } diff --git a/src/lib/redis/lua-scripts.ts b/src/lib/redis/lua-scripts.ts index c0da78e89..7feb1a763 100644 --- a/src/lib/redis/lua-scripts.ts +++ b/src/lib/redis/lua-scripts.ts @@ -376,3 +376,376 @@ end return tostring(total) `; + +/** + * Atomically read a tenant-scoped session binding and reconcile it with the + * legacy session-only mirror during a rolling upgrade. + * + * KEYS[1]: canonical binding hash + * KEYS[2]: legacy provider string + * KEYS[3]: legacy key owner string + * ARGV[1]: current key id + * ARGV[2]: generation to use when initializing/upgrading + * ARGV[3]: binding TTL in seconds + * + * Return: + * - {"ok", source, generation, providerIdOrEmpty} + * - {"conflict", reason} + */ +export const READ_OR_RECONCILE_SESSION_BINDING = ` +local binding_key = KEYS[1] +local legacy_provider_key = KEYS[2] +local legacy_owner_key = KEYS[3] + +local current_key_id = ARGV[1] +local new_generation = ARGV[2] +local ttl = tonumber(ARGV[3]) + +local function is_positive_integer(value) + local parsed = tonumber(value) + return parsed and parsed > 0 and parsed == math.floor(parsed) +end + +if not ttl or ttl <= 0 or current_key_id == '' or new_generation == '' then + return {'conflict', 'invalid_input'} +end + +local legacy_provider = redis.call('GET', legacy_provider_key) +local legacy_owner = redis.call('GET', legacy_owner_key) +local binding_exists = redis.call('EXISTS', binding_key) == 1 + +if binding_exists then + local binding = redis.call('HMGET', binding_key, 'key_id', 'generation', 'provider_id') + local binding_key_id = binding[1] + local generation = binding[2] + local provider_id = binding[3] + + if not binding_key_id or not generation or binding_key_id == '' or generation == '' then + return {'conflict', 'canonical_corrupt'} + end + if binding_key_id ~= current_key_id then + return {'conflict', 'canonical_key_mismatch'} + end + if not legacy_owner then + return {'conflict', 'mirror_missing'} + end + if legacy_owner ~= current_key_id then + return {'conflict', 'foreign_legacy_owner'} + end + + if provider_id then + if not is_positive_integer(provider_id) then + return {'conflict', 'canonical_corrupt'} + end + if legacy_provider ~= provider_id then + return {'conflict', 'mirror_conflict'} + end + redis.call('EXPIRE', legacy_provider_key, ttl) + elseif legacy_provider then + return {'conflict', 'mirror_conflict'} + end + + redis.call('EXPIRE', binding_key, ttl) + redis.call('EXPIRE', legacy_owner_key, ttl) + return {'ok', 'existing', generation, provider_id or ''} +end + +if legacy_owner and legacy_owner ~= current_key_id then + return {'conflict', 'foreign_legacy_owner'} +end +if legacy_provider and not legacy_owner then + return {'conflict', 'orphan_legacy_provider'} +end +if legacy_provider and not is_positive_integer(legacy_provider) then + return {'conflict', 'invalid_legacy_provider'} +end + +if not legacy_owner and not legacy_provider then + redis.call('HSET', binding_key, 'key_id', current_key_id, 'generation', new_generation) + redis.call('HDEL', binding_key, 'provider_id') + redis.call('EXPIRE', binding_key, ttl) + redis.call('SETEX', legacy_owner_key, ttl, current_key_id) + return {'ok', 'created', new_generation, ''} +end + +-- At this point the legacy owner is current_key_id. The provider may be absent, +-- which is the valid null-binding mirror used by a fresh session. +redis.call('HSET', binding_key, 'key_id', current_key_id, 'generation', new_generation) +if legacy_provider then + redis.call('HSET', binding_key, 'provider_id', legacy_provider) + redis.call('EXPIRE', legacy_provider_key, ttl) +else + redis.call('HDEL', binding_key, 'provider_id') +end +redis.call('EXPIRE', binding_key, ttl) +redis.call('EXPIRE', legacy_owner_key, ttl) +return {'ok', 'legacy_upgraded', new_generation, legacy_provider or ''} +`; + +/** + * Compare-and-set a provider on an existing versioned session binding. + * The canonical hash and both legacy mirrors are validated before dual-write. + * Missing canonical state is always a conflict and is never initialized here. + * + * KEYS[1]: canonical binding hash + * KEYS[2]: legacy provider string + * KEYS[3]: legacy key owner string + * ARGV[1]: current key id + * ARGV[2]: expected generation + * ARGV[3]: next generation + * ARGV[4]: next provider id + * ARGV[5]: binding TTL in seconds + */ +export const CAS_SESSION_BINDING = ` +local binding_key = KEYS[1] +local legacy_provider_key = KEYS[2] +local legacy_owner_key = KEYS[3] + +local current_key_id = ARGV[1] +local expected_generation = ARGV[2] +local next_generation = ARGV[3] +local next_provider_id = ARGV[4] +local ttl = tonumber(ARGV[5]) + +local function is_positive_integer(value) + local parsed = tonumber(value) + return parsed and parsed > 0 and parsed == math.floor(parsed) +end + +if not ttl or ttl <= 0 or current_key_id == '' or expected_generation == '' or + next_generation == '' or not is_positive_integer(next_provider_id) then + return {'conflict', 'invalid_input'} +end +if redis.call('EXISTS', binding_key) == 0 then + return {'conflict', 'canonical_missing'} +end + +local binding = redis.call('HMGET', binding_key, 'key_id', 'generation', 'provider_id') +local binding_key_id = binding[1] +local generation = binding[2] +local current_provider_id = binding[3] + +if not binding_key_id or not generation or binding_key_id == '' or generation == '' then + return {'conflict', 'canonical_corrupt'} +end +if binding_key_id ~= current_key_id then + return {'conflict', 'canonical_key_mismatch'} +end +if generation ~= expected_generation then + return {'conflict', 'generation_mismatch'} +end + +local legacy_owner = redis.call('GET', legacy_owner_key) +local legacy_provider = redis.call('GET', legacy_provider_key) +if current_provider_id and not is_positive_integer(current_provider_id) then + return {'conflict', 'canonical_corrupt'} +end +if legacy_provider and not is_positive_integer(legacy_provider) then + return {'conflict', 'invalid_legacy_provider'} +end +if not legacy_owner then + return {'conflict', 'mirror_missing'} +end +if legacy_owner ~= current_key_id then + return {'conflict', 'foreign_legacy_owner'} +end +if current_provider_id then + if legacy_provider ~= current_provider_id then + return {'conflict', 'mirror_conflict'} + end +elseif legacy_provider then + return {'conflict', 'mirror_conflict'} +end + +redis.call('HSET', binding_key, + 'key_id', current_key_id, + 'generation', next_generation, + 'provider_id', next_provider_id) +redis.call('EXPIRE', binding_key, ttl) +redis.call('SETEX', legacy_owner_key, ttl, current_key_id) +redis.call('SETEX', legacy_provider_key, ttl, next_provider_id) +return {'ok', 'updated', next_generation, next_provider_id} +`; + +/** + * Compare-and-clear an existing versioned session binding. A cooldown marker + * may be written in the same transaction when clearing a timed-out provider. + * + * KEYS[1]: canonical binding hash + * KEYS[2]: legacy provider string + * KEYS[3]: legacy key owner string + * KEYS[4]: tenant-scoped cooldown key (unused when cooldown TTL is zero) + * ARGV[1]: current key id + * ARGV[2]: expected generation + * ARGV[3]: next generation + * ARGV[4]: expected provider id, or empty for a null binding + * ARGV[5]: binding TTL in seconds + * ARGV[6]: cooldown provider id, or empty + * ARGV[7]: cooldown TTL in seconds, or zero + */ +export const CLEAR_SESSION_BINDING = ` +local binding_key = KEYS[1] +local legacy_provider_key = KEYS[2] +local legacy_owner_key = KEYS[3] +local cooldown_key = KEYS[4] + +local current_key_id = ARGV[1] +local expected_generation = ARGV[2] +local next_generation = ARGV[3] +local expected_provider_id = ARGV[4] +local ttl = tonumber(ARGV[5]) +local cooldown_provider_id = ARGV[6] +local cooldown_ttl = tonumber(ARGV[7]) or 0 + +local function is_positive_integer(value) + local parsed = tonumber(value) + return parsed and parsed > 0 and parsed == math.floor(parsed) +end + +if not ttl or ttl <= 0 or current_key_id == '' or expected_generation == '' or + next_generation == '' or cooldown_ttl < 0 then + return {'conflict', 'invalid_input'} +end +if cooldown_ttl > 0 and + (not is_positive_integer(expected_provider_id) or + cooldown_provider_id ~= expected_provider_id) then + return {'conflict', 'invalid_input'} +end +if redis.call('EXISTS', binding_key) == 0 then + return {'conflict', 'canonical_missing'} +end + +local binding = redis.call('HMGET', binding_key, 'key_id', 'generation', 'provider_id') +local binding_key_id = binding[1] +local generation = binding[2] +local current_provider_id = binding[3] + +if not binding_key_id or not generation or binding_key_id == '' or generation == '' then + return {'conflict', 'canonical_corrupt'} +end +if binding_key_id ~= current_key_id then + return {'conflict', 'canonical_key_mismatch'} +end +if generation ~= expected_generation then + return {'conflict', 'generation_mismatch'} +end +if (current_provider_id or '') ~= expected_provider_id then + return {'conflict', 'provider_mismatch'} +end + +local legacy_owner = redis.call('GET', legacy_owner_key) +local legacy_provider = redis.call('GET', legacy_provider_key) +if current_provider_id and not is_positive_integer(current_provider_id) then + return {'conflict', 'canonical_corrupt'} +end +if legacy_provider and not is_positive_integer(legacy_provider) then + return {'conflict', 'invalid_legacy_provider'} +end +if not legacy_owner then + return {'conflict', 'mirror_missing'} +end +if legacy_owner ~= current_key_id then + return {'conflict', 'foreign_legacy_owner'} +end +if current_provider_id then + if legacy_provider ~= current_provider_id then + return {'conflict', 'mirror_conflict'} + end +elseif legacy_provider then + return {'conflict', 'mirror_conflict'} +end + +redis.call('HSET', binding_key, 'key_id', current_key_id, 'generation', next_generation) +redis.call('HDEL', binding_key, 'provider_id') +redis.call('EXPIRE', binding_key, ttl) +redis.call('SETEX', legacy_owner_key, ttl, current_key_id) +redis.call('DEL', legacy_provider_key) + +if cooldown_ttl > 0 then + redis.call('SETEX', cooldown_key, cooldown_ttl, next_generation) +end + +return {'ok', 'cleared', next_generation, ''} +`; + +/** + * Tenant-authorized administrative termination. Unlike request-level clear, + * this operation intentionally does not compare an old generation. It still + * validates canonical ownership and both legacy mirrors before rotating the + * generation and leaving a null tombstone. + * + * KEYS[1]: canonical binding hash + * KEYS[2]: legacy provider string + * KEYS[3]: legacy key owner string + * ARGV[1]: current key id + * ARGV[2]: next generation + * ARGV[3]: binding TTL in seconds + * ARGV[4]: optional expected provider id for conditional batch termination + */ +export const TERMINATE_SESSION_BINDING = ` +local binding_key = KEYS[1] +local legacy_provider_key = KEYS[2] +local legacy_owner_key = KEYS[3] + +local current_key_id = ARGV[1] +local next_generation = ARGV[2] +local ttl = tonumber(ARGV[3]) +local expected_provider_id = ARGV[4] + +local function is_positive_integer(value) + local parsed = tonumber(value) + return parsed and parsed > 0 and parsed == math.floor(parsed) +end + +if not ttl or ttl <= 0 or current_key_id == '' or next_generation == '' or + (expected_provider_id ~= '' and not is_positive_integer(expected_provider_id)) then + return {'conflict', 'invalid_input'} +end +if redis.call('EXISTS', binding_key) == 0 then + return {'conflict', 'canonical_missing'} +end + +local binding = redis.call('HMGET', binding_key, 'key_id', 'generation', 'provider_id') +local binding_key_id = binding[1] +local generation = binding[2] +local current_provider_id = binding[3] + +if not binding_key_id or not generation or binding_key_id == '' or generation == '' then + return {'conflict', 'canonical_corrupt'} +end +if binding_key_id ~= current_key_id then + return {'conflict', 'canonical_key_mismatch'} +end +if current_provider_id and not is_positive_integer(current_provider_id) then + return {'conflict', 'canonical_corrupt'} +end +if expected_provider_id ~= '' and (current_provider_id or '') ~= expected_provider_id then + return {'conflict', 'provider_mismatch'} +end + +local legacy_owner = redis.call('GET', legacy_owner_key) +local legacy_provider = redis.call('GET', legacy_provider_key) +if legacy_provider and not is_positive_integer(legacy_provider) then + return {'conflict', 'invalid_legacy_provider'} +end +if not legacy_owner then + return {'conflict', 'mirror_missing'} +end +if legacy_owner ~= current_key_id then + return {'conflict', 'foreign_legacy_owner'} +end +if current_provider_id then + if legacy_provider ~= current_provider_id then + return {'conflict', 'mirror_conflict'} + end +elseif legacy_provider then + return {'conflict', 'mirror_conflict'} +end + +redis.call('HSET', binding_key, 'key_id', current_key_id, 'generation', next_generation) +redis.call('HDEL', binding_key, 'provider_id') +redis.call('EXPIRE', binding_key, ttl) +redis.call('SETEX', legacy_owner_key, ttl, current_key_id) +redis.call('DEL', legacy_provider_key) +return {'ok', 'terminated', next_generation, ''} +`; diff --git a/src/lib/redis/session-binding.ts b/src/lib/redis/session-binding.ts new file mode 100644 index 000000000..eb6037a5f --- /dev/null +++ b/src/lib/redis/session-binding.ts @@ -0,0 +1,989 @@ +import "server-only"; + +import { createHash, randomUUID } from "node:crypto"; +import { logger } from "@/lib/logger"; +import { getRedisClient } from "./client"; +import { + CAS_SESSION_BINDING, + CLEAR_SESSION_BINDING, + READ_OR_RECONCILE_SESSION_BINDING, + TERMINATE_SESSION_BINDING, +} from "./lua-scripts"; + +export const DEFAULT_SESSION_BINDING_TTL_SECONDS = 300; +const CAPABILITY_PROBE_TIMEOUT_MS = 5_000; + +export type VersionedBindingCapabilityState = "unknown" | "available" | "unavailable"; + +export type SessionBindingConflictReason = + | "canonical_corrupt" + | "canonical_key_mismatch" + | "canonical_missing" + | "canonical_exists" + | "foreign_legacy_owner" + | "generation_mismatch" + | "invalid_input" + | "invalid_legacy_provider" + | "mirror_conflict" + | "mirror_missing" + | "orphan_legacy_provider" + | "provider_mismatch" + | "unknown_conflict"; + +export type SessionBindingUnavailableReason = + | "capability_probe_failed" + | "capability_unavailable" + | "connection_changed" + | "operation_failed" + | "redis_not_ready"; + +export interface SessionBindingSnapshot { + sessionId: string; + keyId: number; + providerId: number | null; + generation: string; +} + +export interface SessionBindingKeys { + canonical: string; + legacyProvider: string; + legacyOwner: string; +} + +export interface SessionBindingRedisClient { + readonly status: string; + eval(script: string, numberOfKeys: number, ...args: Array): Promise; + evalsha?(sha1: string, numberOfKeys: number, ...args: Array): Promise; + get(key: string): Promise; + del(...keys: string[]): Promise; + exists(key: string): Promise; + expire(key: string, ttlSeconds: number): Promise; + on(event: string, listener: (...args: unknown[]) => void): unknown; + off?(event: string, listener: (...args: unknown[]) => void): unknown; + set( + key: string, + value: string, + expiryMode: "EX", + ttlSeconds: number, + condition: "NX" + ): Promise; + setex(key: string, ttlSeconds: number, value: string): Promise; +} + +export interface ReadOrReconcileSessionBindingInput { + sessionId: string; + keyId: number; + ttlSeconds?: number; + redis?: SessionBindingRedisClient; +} + +export interface CompareAndSetSessionBindingInput extends ReadOrReconcileSessionBindingInput { + expectedGeneration: string; + providerId: number; +} + +export interface ClearSessionBindingInput extends ReadOrReconcileSessionBindingInput { + expectedGeneration: string; + expectedProviderId: number | null; + cooldownTtlSeconds?: number; +} + +export interface TerminateSessionBindingInput extends ReadOrReconcileSessionBindingInput { + expectedProviderId?: number; +} + +export type LegacySessionBindingMutation = + | { type: "inspect" } + | { type: "refresh" } + | { type: "bind_if_absent"; providerId: number } + | { type: "set"; providerId: number } + | { + type: "clear"; + expectedProviderId?: number | null; + expectedProviderIds?: readonly number[]; + } + | { type: "terminate"; expectedProviderIds?: readonly number[] }; + +export interface LegacySessionBindingMutationInput extends ReadOrReconcileSessionBindingInput { + mutation: LegacySessionBindingMutation; +} + +export type LegacySessionBindingMutationResult = + | { + status: "ok"; + changed: boolean; + providerId: number | null; + } + | SessionBindingConflictResult + | SessionBindingUnavailableResult; + +export interface SessionProviderCooldownInput { + sessionId: string; + keyId: number; + providerId: number; + redis?: SessionBindingRedisClient; +} + +export interface SessionBindingOkResult { + status: "ok"; + snapshot: SessionBindingSnapshot; + legacyFallbackAllowed: false; + source: "created" | "existing" | "legacy_upgraded" | "updated" | "cleared" | "terminated"; +} + +export interface SessionBindingConflictResult { + status: "conflict"; + reason: SessionBindingConflictReason; + legacyFallbackAllowed: false; +} + +export interface SessionBindingUnavailableResult { + status: "unavailable"; + reason: SessionBindingUnavailableReason; + capabilityState: VersionedBindingCapabilityState; + legacyFallbackAllowed: boolean; +} + +export type SessionBindingResult = + | SessionBindingOkResult + | SessionBindingConflictResult + | SessionBindingUnavailableResult; +type SessionBindingFailureResult = SessionBindingConflictResult | SessionBindingUnavailableResult; + +export type SessionProviderCooldownResult = + | { + status: "ok"; + coolingDown: boolean; + legacyFallbackAllowed: false; + } + | SessionBindingConflictResult + | SessionBindingUnavailableResult; + +interface CapabilityListeners { + close: (...args: unknown[]) => void; + connect: (...args: unknown[]) => void; + end: (...args: unknown[]) => void; + ready: (...args: unknown[]) => void; + reconnecting: (...args: unknown[]) => void; +} + +interface ReadyClient { + redis: SessionBindingRedisClient; + epoch: number; +} + +const READ_SOURCES = new Set(["created", "existing", "legacy_upgraded"]); +const MUTATION_SOURCES = new Set(["updated", "cleared"]); +const CONFLICT_REASONS = new Set([ + "canonical_corrupt", + "canonical_exists", + "canonical_key_mismatch", + "canonical_missing", + "foreign_legacy_owner", + "generation_mismatch", + "invalid_input", + "invalid_legacy_provider", + "mirror_conflict", + "mirror_missing", + "orphan_legacy_provider", + "provider_mismatch", +]); + +let capabilityClient: SessionBindingRedisClient | null = null; +let capabilityState: VersionedBindingCapabilityState = "unknown"; +let capabilityEpoch = 0; +let capabilityProbe: Promise | null = null; +let capabilityListeners: CapabilityListeners | null = null; +const scriptSha1Cache = new Map(); + +function namespacedKey(namespace: string | undefined, key: string): string { + return namespace ? `${namespace}:${key}` : key; +} + +function bindingHashTag(sessionId: string, keyId: number): string { + return createHash("sha256").update(`${keyId}\0${sessionId}`).digest("hex"); +} + +export function buildCanonicalSessionBindingKey( + sessionId: string, + keyId: number, + namespace?: string +): string { + return namespacedKey( + namespace, + `session-binding:v1:{${bindingHashTag(sessionId, keyId)}}:binding` + ); +} + +export function buildLegacySessionProviderKey(sessionId: string, namespace?: string): string { + return namespacedKey(namespace, `session:${sessionId}:provider`); +} + +export function buildLegacySessionOwnerKey(sessionId: string, namespace?: string): string { + return namespacedKey(namespace, `session:${sessionId}:key`); +} + +export function buildSessionProviderCooldownKey( + sessionId: string, + keyId: number, + providerId: number, + namespace?: string +): string { + return namespacedKey( + namespace, + `session-binding:v1:{${bindingHashTag(sessionId, keyId)}}:provider:${providerId}:cooldown` + ); +} + +export function buildSessionBindingKeys( + sessionId: string, + keyId: number, + namespace?: string +): SessionBindingKeys { + return { + canonical: buildCanonicalSessionBindingKey(sessionId, keyId, namespace), + legacyProvider: buildLegacySessionProviderKey(sessionId, namespace), + legacyOwner: buildLegacySessionOwnerKey(sessionId, namespace), + }; +} + +function isPositiveInteger(value: number): boolean { + return Number.isSafeInteger(value) && value > 0; +} + +function isValidIdentity(sessionId: string, keyId: number, ttlSeconds: number): boolean { + return sessionId.length > 0 && isPositiveInteger(keyId) && isPositiveInteger(ttlSeconds); +} + +function conflict(reason: SessionBindingConflictReason): SessionBindingConflictResult { + return { status: "conflict", reason, legacyFallbackAllowed: false }; +} + +function unavailable(reason: SessionBindingUnavailableReason): SessionBindingUnavailableResult { + return { + status: "unavailable", + reason, + capabilityState, + legacyFallbackAllowed: + reason === "capability_probe_failed" || + reason === "capability_unavailable" || + reason === "redis_not_ready", + }; +} + +function normalizeEvalResult(raw: unknown): string[] { + if (!Array.isArray(raw) || raw.length < 2) { + throw new Error("Invalid session binding Lua result"); + } + return raw.map((value) => { + if (typeof value === "string" || typeof value === "number") { + return String(value); + } + if (Buffer.isBuffer(value)) { + return value.toString("utf8"); + } + throw new Error("Invalid value in session binding Lua result"); + }); +} + +function parseProviderId(raw: string): number | null { + if (raw === "") return null; + const providerId = Number(raw); + if (!isPositiveInteger(providerId)) { + throw new Error("Invalid provider id in session binding Lua result"); + } + return providerId; +} + +function parseBindingResult( + raw: unknown, + identity: { sessionId: string; keyId: number }, + allowedSources: Set +): SessionBindingResult { + const values = normalizeEvalResult(raw); + if (values[0] === "conflict") { + const reason = CONFLICT_REASONS.has(values[1] as SessionBindingConflictReason) + ? (values[1] as SessionBindingConflictReason) + : "unknown_conflict"; + return conflict(reason); + } + if (values[0] !== "ok" || values.length < 4 || !allowedSources.has(values[1])) { + throw new Error("Unexpected session binding Lua result"); + } + + const generation = values[2]; + if (!generation) { + throw new Error("Missing generation in session binding Lua result"); + } + + return { + status: "ok", + source: values[1] as SessionBindingOkResult["source"], + snapshot: { + ...identity, + generation, + providerId: parseProviderId(values[3]), + }, + legacyFallbackAllowed: false, + }; +} + +function detachCapabilityListeners(): void { + if (!capabilityClient || !capabilityListeners || !capabilityClient.off) return; + capabilityClient.off("close", capabilityListeners.close); + capabilityClient.off("connect", capabilityListeners.connect); + capabilityClient.off("end", capabilityListeners.end); + capabilityClient.off("ready", capabilityListeners.ready); + capabilityClient.off("reconnecting", capabilityListeners.reconnecting); +} + +function resetCapabilityForConnection(client: SessionBindingRedisClient): void { + if (capabilityClient !== client) return; + capabilityEpoch += 1; + capabilityState = "unknown"; + capabilityProbe = null; +} + +function attachCapabilityClient(client: SessionBindingRedisClient): void { + if (capabilityClient === client) return; + + detachCapabilityListeners(); + capabilityClient = client; + capabilityEpoch += 1; + capabilityState = "unknown"; + capabilityProbe = null; + + const listeners: CapabilityListeners = { + close: () => resetCapabilityForConnection(client), + connect: () => resetCapabilityForConnection(client), + end: () => resetCapabilityForConnection(client), + ready: () => { + resetCapabilityForConnection(client); + void ensureVersionedBindingCapability(client); + }, + reconnecting: () => resetCapabilityForConnection(client), + }; + capabilityListeners = listeners; + client.on("close", listeners.close); + client.on("connect", listeners.connect); + client.on("end", listeners.end); + client.on("ready", listeners.ready); + client.on("reconnecting", listeners.reconnecting); +} + +function currentRedisClient( + override?: SessionBindingRedisClient +): SessionBindingRedisClient | null { + if (override) return override; + return getRedisClient({ allowWhenRateLimitDisabled: true }) as SessionBindingRedisClient | null; +} + +function scriptSha1(script: string): string { + const cached = scriptSha1Cache.get(script); + if (cached) return cached; + const digest = createHash("sha1").update(script).digest("hex"); + scriptSha1Cache.set(script, digest); + return digest; +} + +async function evalBindingScript( + redis: SessionBindingRedisClient, + script: string, + numberOfKeys: number, + ...args: Array +): Promise { + if (!redis.evalsha) return redis.eval(script, numberOfKeys, ...args); + try { + return await redis.evalsha(scriptSha1(script), numberOfKeys, ...args); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!/NOSCRIPT/i.test(message)) throw error; + return redis.eval(script, numberOfKeys, ...args); + } +} + +function markCapabilityUnavailable( + client: SessionBindingRedisClient, + epoch: number, + error: unknown +): void { + if (capabilityClient !== client || capabilityEpoch !== epoch) return; + capabilityState = "unavailable"; + capabilityProbe = null; + logger.warn("Versioned session binding Redis capability is unavailable", { + error: error instanceof Error ? error.message : String(error), + }); +} + +function isCapabilityError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return /CROSSSLOT|NOPERM|unknown command|EVAL.*(?:disabled|not allowed)|script execution disabled|Lua scripts? (?:are )?disabled/i.test( + message + ); +} + +function isBindingDataError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return ( + message.startsWith("Invalid session binding Lua result") || + message.startsWith("Invalid value in session binding Lua result") || + message.startsWith("Unexpected session binding Lua result") || + message.startsWith("Missing generation in session binding Lua result") || + message.startsWith("Invalid provider id in session binding Lua result") || + message.includes("WRONGTYPE") + ); +} + +function handleOperationError(ready: ReadyClient, error: unknown): SessionBindingFailureResult { + if (isCapabilityError(error)) { + markCapabilityUnavailable(ready.redis, ready.epoch, error); + return unavailable("capability_unavailable"); + } + + if (isBindingDataError(error)) { + logger.warn("Versioned session binding data is invalid", { + error: error instanceof Error ? error.message : String(error), + }); + return conflict("canonical_corrupt"); + } + + logger.warn("Versioned session binding operation failed", { + error: error instanceof Error ? error.message : String(error), + }); + return unavailable("operation_failed"); +} + +async function runCapabilityProbe( + client: SessionBindingRedisClient, + epoch: number +): Promise { + const deadlineAt = Date.now() + CAPABILITY_PROBE_TIMEOUT_MS; + const namespace = `session-binding-capability-probe:${randomUUID()}`; + const sessionId = "probe-session"; + const keyId = 1; + const providerId = 1; + const ttlSeconds = 60; + const keys = buildSessionBindingKeys(sessionId, keyId, namespace); + const cooldownKey = buildSessionProviderCooldownKey(sessionId, keyId, providerId, namespace); + const cleanupKeys = [keys.canonical, keys.legacyProvider, keys.legacyOwner, cooldownKey]; + const initialGeneration = randomUUID(); + const boundGeneration = randomUUID(); + const clearedGeneration = randomUUID(); + let operationsSucceeded = false; + let cleanupSucceeded = false; + + try { + const read = parseBindingResult( + await withCapabilityProbeDeadline( + client.eval( + READ_OR_RECONCILE_SESSION_BINDING, + 3, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + keyId.toString(), + initialGeneration, + ttlSeconds.toString() + ), + deadlineAt + ), + { sessionId, keyId }, + READ_SOURCES + ); + if (read.status !== "ok" || read.source !== "created") { + throw new Error("Session binding capability probe reconcile failed"); + } + + const updated = parseBindingResult( + await withCapabilityProbeDeadline( + client.eval( + CAS_SESSION_BINDING, + 3, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + keyId.toString(), + read.snapshot.generation, + boundGeneration, + providerId.toString(), + ttlSeconds.toString() + ), + deadlineAt + ), + { sessionId, keyId }, + MUTATION_SOURCES + ); + if (updated.status !== "ok" || updated.source !== "updated") { + throw new Error("Session binding capability probe update failed"); + } + + const cleared = parseBindingResult( + await withCapabilityProbeDeadline( + client.eval( + CLEAR_SESSION_BINDING, + 4, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + cooldownKey, + keyId.toString(), + updated.snapshot.generation, + clearedGeneration, + providerId.toString(), + ttlSeconds.toString(), + providerId.toString(), + ttlSeconds.toString() + ), + deadlineAt + ), + { sessionId, keyId }, + MUTATION_SOURCES + ); + if (cleared.status !== "ok" || cleared.source !== "cleared") { + throw new Error("Session binding capability probe clear failed"); + } + + const cooldownGeneration = await withCapabilityProbeDeadline( + client.get(cooldownKey), + deadlineAt + ); + if (cooldownGeneration !== cleared.snapshot.generation) { + throw new Error("Session binding capability probe cooldown failed"); + } + operationsSucceeded = capabilityClient === client && capabilityEpoch === epoch; + } catch (error) { + logger.warn("Versioned session binding Redis capability probe failed", { + error: error instanceof Error ? error.message : String(error), + }); + } finally { + try { + await withCapabilityProbeDeadline( + Promise.all(cleanupKeys.map((key) => client.del(key))), + deadlineAt + ); + cleanupSucceeded = true; + } catch (error) { + logger.warn("Versioned session binding Redis capability probe cleanup failed", { + error: error instanceof Error ? error.message : String(error), + }); + } + } + + return operationsSucceeded && cleanupSucceeded; +} + +async function withCapabilityProbeDeadline( + operation: Promise, + deadlineAt: number +): Promise { + const remainingMs = deadlineAt - Date.now(); + if (remainingMs <= 0) throw new Error("Session binding capability probe deadline exceeded"); + + let timeout: ReturnType | undefined; + const deadline = new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new Error("Session binding capability probe deadline exceeded")), + remainingMs + ); + timeout.unref?.(); + }); + try { + return await Promise.race([operation, deadline]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +export function getVersionedBindingCapabilityState(): VersionedBindingCapabilityState { + return capabilityState; +} + +export async function ensureVersionedBindingCapability( + redisOverride?: SessionBindingRedisClient +): Promise { + const redis = currentRedisClient(redisOverride); + if (!redis) return "unknown"; + + attachCapabilityClient(redis); + if (redis.status !== "ready") return capabilityState; + if (capabilityState !== "unknown") return capabilityState; + if (capabilityProbe) return capabilityProbe; + + const epoch = capabilityEpoch; + const probe = (async () => { + const supported = await runCapabilityProbe(redis, epoch); + if (capabilityClient !== redis || capabilityEpoch !== epoch) { + return capabilityState; + } + capabilityState = supported ? "available" : "unavailable"; + return capabilityState; + })(); + capabilityProbe = probe; + + try { + return await probe; + } finally { + if (capabilityProbe === probe) capabilityProbe = null; + } +} + +async function readyVersionedClient( + redisOverride?: SessionBindingRedisClient +): Promise { + const redis = currentRedisClient(redisOverride); + if (!redis) return unavailable("redis_not_ready"); + + attachCapabilityClient(redis); + if (redis.status !== "ready") return unavailable("redis_not_ready"); + + const state = await ensureVersionedBindingCapability(redis); + if (state !== "available") { + return unavailable( + state === "unavailable" ? "capability_unavailable" : "capability_probe_failed" + ); + } + return { redis, epoch: capabilityEpoch }; +} + +function connectionIsCurrent(ready: ReadyClient): boolean { + return capabilityClient === ready.redis && capabilityEpoch === ready.epoch; +} + +export async function readOrReconcileSessionBinding( + input: ReadOrReconcileSessionBindingInput +): Promise { + const ttlSeconds = input.ttlSeconds ?? DEFAULT_SESSION_BINDING_TTL_SECONDS; + if (!isValidIdentity(input.sessionId, input.keyId, ttlSeconds)) { + return conflict("invalid_input"); + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + const keys = buildSessionBindingKeys(input.sessionId, input.keyId); + try { + const raw = await evalBindingScript( + ready.redis, + READ_OR_RECONCILE_SESSION_BINDING, + 3, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + input.keyId.toString(), + randomUUID(), + ttlSeconds.toString() + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return parseBindingResult( + raw, + { sessionId: input.sessionId, keyId: input.keyId }, + READ_SOURCES + ); + } catch (error) { + return handleOperationError(ready, error); + } +} + +export async function refreshSessionBinding( + input: ReadOrReconcileSessionBindingInput +): Promise { + return readOrReconcileSessionBinding(input); +} + +export async function compareAndSetSessionBinding( + input: CompareAndSetSessionBindingInput +): Promise { + const ttlSeconds = input.ttlSeconds ?? DEFAULT_SESSION_BINDING_TTL_SECONDS; + if ( + !isValidIdentity(input.sessionId, input.keyId, ttlSeconds) || + !input.expectedGeneration || + !isPositiveInteger(input.providerId) + ) { + return conflict("invalid_input"); + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + const keys = buildSessionBindingKeys(input.sessionId, input.keyId); + try { + const raw = await evalBindingScript( + ready.redis, + CAS_SESSION_BINDING, + 3, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + input.keyId.toString(), + input.expectedGeneration, + randomUUID(), + input.providerId.toString(), + ttlSeconds.toString() + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return parseBindingResult( + raw, + { sessionId: input.sessionId, keyId: input.keyId }, + MUTATION_SOURCES + ); + } catch (error) { + return handleOperationError(ready, error); + } +} + +export async function clearSessionBinding( + input: ClearSessionBindingInput +): Promise { + const ttlSeconds = input.ttlSeconds ?? DEFAULT_SESSION_BINDING_TTL_SECONDS; + const cooldownTtlSeconds = input.cooldownTtlSeconds ?? 0; + if ( + !isValidIdentity(input.sessionId, input.keyId, ttlSeconds) || + !input.expectedGeneration || + (input.expectedProviderId !== null && !isPositiveInteger(input.expectedProviderId)) || + !Number.isSafeInteger(cooldownTtlSeconds) || + cooldownTtlSeconds < 0 || + (cooldownTtlSeconds > 0 && input.expectedProviderId === null) + ) { + return conflict("invalid_input"); + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + const keys = buildSessionBindingKeys(input.sessionId, input.keyId); + const cooldownKey = + input.expectedProviderId === null + ? keys.canonical + : buildSessionProviderCooldownKey(input.sessionId, input.keyId, input.expectedProviderId); + const expectedProviderId = input.expectedProviderId?.toString() ?? ""; + + try { + const raw = await evalBindingScript( + ready.redis, + CLEAR_SESSION_BINDING, + 4, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + cooldownKey, + input.keyId.toString(), + input.expectedGeneration, + randomUUID(), + expectedProviderId, + ttlSeconds.toString(), + cooldownTtlSeconds > 0 ? expectedProviderId : "", + cooldownTtlSeconds.toString() + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return parseBindingResult( + raw, + { sessionId: input.sessionId, keyId: input.keyId }, + MUTATION_SOURCES + ); + } catch (error) { + return handleOperationError(ready, error); + } +} + +export async function terminateSessionBinding( + input: TerminateSessionBindingInput +): Promise { + const ttlSeconds = input.ttlSeconds ?? DEFAULT_SESSION_BINDING_TTL_SECONDS; + if ( + !isValidIdentity(input.sessionId, input.keyId, ttlSeconds) || + (input.expectedProviderId !== undefined && !isPositiveInteger(input.expectedProviderId)) + ) { + return conflict("invalid_input"); + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + const keys = buildSessionBindingKeys(input.sessionId, input.keyId); + try { + const raw = await evalBindingScript( + ready.redis, + TERMINATE_SESSION_BINDING, + 3, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + input.keyId.toString(), + randomUUID(), + ttlSeconds.toString(), + input.expectedProviderId?.toString() ?? "" + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return parseBindingResult( + raw, + { sessionId: input.sessionId, keyId: input.keyId }, + new Set(["terminated"]) + ); + } catch (error) { + return handleOperationError(ready, error); + } +} + +function parseLegacyProviderId(raw: string | null): number | null | undefined { + if (raw === null) return null; + const providerId = Number(raw); + return isPositiveInteger(providerId) ? providerId : undefined; +} + +export async function mutateLegacySessionBindingSafely( + input: LegacySessionBindingMutationInput +): Promise { + const ttlSeconds = input.ttlSeconds ?? DEFAULT_SESSION_BINDING_TTL_SECONDS; + if (!isValidIdentity(input.sessionId, input.keyId, ttlSeconds)) { + return conflict("invalid_input"); + } + + const providerFromMutation = + input.mutation.type === "bind_if_absent" || input.mutation.type === "set" + ? input.mutation.providerId + : null; + if (providerFromMutation !== null && !isPositiveInteger(providerFromMutation)) { + return conflict("invalid_input"); + } + if ( + input.mutation.type === "terminate" && + input.mutation.expectedProviderIds?.some((providerId) => !isPositiveInteger(providerId)) + ) { + return conflict("invalid_input"); + } + if ( + input.mutation.type === "clear" && + ((input.mutation.expectedProviderId != null && + !isPositiveInteger(input.mutation.expectedProviderId)) || + input.mutation.expectedProviderIds?.some((providerId) => !isPositiveInteger(providerId)) || + (input.mutation.expectedProviderId != null && + input.mutation.expectedProviderIds !== undefined)) + ) { + return conflict("invalid_input"); + } + + const redis = currentRedisClient(input.redis); + if (!redis || redis.status !== "ready") return unavailable("redis_not_ready"); + + const keys = buildSessionBindingKeys(input.sessionId, input.keyId); + try { + if ((await redis.exists(keys.canonical)) > 0) { + return conflict("canonical_exists"); + } + + let [legacyOwner, legacyProviderRaw] = await Promise.all([ + redis.get(keys.legacyOwner), + redis.get(keys.legacyProvider), + ]); + if (legacyOwner === null) { + if (legacyProviderRaw !== null) return conflict("orphan_legacy_provider"); + await redis.set(keys.legacyOwner, input.keyId.toString(), "EX", ttlSeconds, "NX"); + [legacyOwner, legacyProviderRaw] = await Promise.all([ + redis.get(keys.legacyOwner), + redis.get(keys.legacyProvider), + ]); + } + + if (legacyOwner !== input.keyId.toString()) return conflict("foreign_legacy_owner"); + const legacyProvider = parseLegacyProviderId(legacyProviderRaw); + if (legacyProvider === undefined) return conflict("invalid_legacy_provider"); + if ((await redis.exists(keys.canonical)) > 0) return conflict("canonical_exists"); + + switch (input.mutation.type) { + case "inspect": + return { status: "ok", changed: false, providerId: legacyProvider }; + case "refresh": + await redis.expire(keys.legacyOwner, ttlSeconds); + if (legacyProvider !== null) await redis.expire(keys.legacyProvider, ttlSeconds); + return { status: "ok", changed: false, providerId: legacyProvider }; + case "bind_if_absent": { + if (legacyProvider !== null) { + return { status: "ok", changed: false, providerId: legacyProvider }; + } + const result = await redis.set( + keys.legacyProvider, + input.mutation.providerId.toString(), + "EX", + ttlSeconds, + "NX" + ); + await redis.expire(keys.legacyOwner, ttlSeconds); + if (result === "OK") { + return { status: "ok", changed: true, providerId: input.mutation.providerId }; + } + const concurrentProvider = parseLegacyProviderId(await redis.get(keys.legacyProvider)); + if (concurrentProvider === undefined) return conflict("invalid_legacy_provider"); + return { status: "ok", changed: false, providerId: concurrentProvider }; + } + case "set": + await redis.setex(keys.legacyProvider, ttlSeconds, input.mutation.providerId.toString()); + await redis.expire(keys.legacyOwner, ttlSeconds); + return { status: "ok", changed: true, providerId: input.mutation.providerId }; + case "clear": + if ( + (input.mutation.expectedProviderId != null && + legacyProvider !== input.mutation.expectedProviderId) || + (input.mutation.expectedProviderIds && + (legacyProvider === null || + !input.mutation.expectedProviderIds.includes(legacyProvider))) + ) { + return conflict("provider_mismatch"); + } + if (legacyProvider === null) { + return { status: "ok", changed: false, providerId: null }; + } + await redis.del(keys.legacyProvider); + await redis.expire(keys.legacyOwner, ttlSeconds); + return { status: "ok", changed: true, providerId: null }; + case "terminate": + if ( + input.mutation.expectedProviderIds && + (legacyProvider === null || !input.mutation.expectedProviderIds.includes(legacyProvider)) + ) { + return conflict("provider_mismatch"); + } + if (legacyProvider !== null) await redis.del(keys.legacyProvider); + await redis.del(keys.legacyOwner); + return { status: "ok", changed: legacyProvider !== null, providerId: null }; + } + } catch (error) { + logger.warn("Legacy session binding mutation failed", { + error: error instanceof Error ? error.message : String(error), + }); + return unavailable("operation_failed"); + } +} + +export async function isSessionProviderCoolingDown( + input: SessionProviderCooldownInput +): Promise { + if ( + input.sessionId.length === 0 || + !isPositiveInteger(input.keyId) || + !isPositiveInteger(input.providerId) + ) { + return conflict("invalid_input"); + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + try { + const value = await ready.redis.get( + buildSessionProviderCooldownKey(input.sessionId, input.keyId, input.providerId) + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return { status: "ok", coolingDown: value !== null, legacyFallbackAllowed: false }; + } catch (error) { + return handleOperationError(ready, error); + } +} + +export function resetVersionedBindingCapabilityForTests(): void { + detachCapabilityListeners(); + capabilityClient = null; + capabilityState = "unknown"; + capabilityEpoch = 0; + capabilityProbe = null; + capabilityListeners = null; +} diff --git a/src/lib/session-manager.ts b/src/lib/session-manager.ts index d44272252..b47b9a3bf 100644 --- a/src/lib/session-manager.ts +++ b/src/lib/session-manager.ts @@ -31,6 +31,20 @@ import { getKeyActiveSessionsKey, getUserActiveSessionsKey, } from "./redis/active-session-keys"; +import { + clearSessionBinding as clearVersionedSessionBinding, + compareAndSetSessionBinding, + mutateLegacySessionBindingSafely, + readOrReconcileSessionBinding, + isSessionProviderCoolingDown as readSessionProviderCooldown, + getVersionedBindingCapabilityState as readVersionedBindingCapabilityState, + type SessionBindingResult, + type SessionBindingSnapshot, + type SessionBindingUnavailableResult, + type SessionProviderCooldownResult, + terminateSessionBinding as terminateVersionedSessionBinding, + type VersionedBindingCapabilityState, +} from "./redis/session-binding"; import { SessionTracker } from "./session-tracker"; const RESERVED_INTERNAL_HEADER_SET = new Set( @@ -42,6 +56,15 @@ function isReservedInternalHeader(name: string): boolean { return lowerName.startsWith("x-cch-") || RESERVED_INTERNAL_HEADER_SET.has(lowerName); } +function redisUnavailableBindingResult(): SessionBindingUnavailableResult { + return { + status: "unavailable", + reason: "redis_not_ready", + capabilityState: readVersionedBindingCapabilityState(), + legacyFallbackAllowed: true, + }; +} + /** * 将已脱敏的 header 文本解析为可序列化对象（用于写入 Session 元信息）。 */ @@ -562,14 +585,31 @@ export class SessionManager { if (!redis || redis.status !== "ready") return; try { + const binding = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status !== "ok") { + if (!binding.legacyFallbackAllowed) return; + const legacy = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "inspect" }, + }); + if (legacy.status !== "ok") return; + } + const pipeline = redis.pipeline(); const hashKey = `hash:${contentHash}:session`; // 存储映射关系 pipeline.setex(hashKey, SessionManager.SESSION_TTL, sessionId); - // 初始化 session 元数据 - pipeline.setex(`session:${sessionId}:key`, SessionManager.SESSION_TTL, keyId.toString()); + // Initialize non-binding session metadata after tenant ownership is proven. pipeline.setex( `session:${sessionId}:last_seen`, SessionManager.SESSION_TTL, @@ -587,16 +627,31 @@ export class SessionManager { /** * 刷新 session TTL（滑动窗口） */ - private static async refreshSessionTTL(sessionId: string, _keyId?: number | null): Promise { + private static async refreshSessionTTL(sessionId: string, keyId?: number | null): Promise { const redis = getRedisClient(); if (!redis || redis.status !== "ready") return; try { const pipeline = redis.pipeline(); - - // TTL 刷新不能改写 session 归属；这里只延长已有 key/provider 绑定的存活时间。 - pipeline.expire(`session:${sessionId}:key`, SessionManager.SESSION_TTL); - pipeline.expire(`session:${sessionId}:provider`, SessionManager.SESSION_TTL); + // Provider selection performs the authoritative binding reconcile. Keep + // this path limited to session activity metadata so a request does not + // pay for a second full binding Lua round trip before selection. + if (keyId != null && readVersionedBindingCapabilityState() === "unavailable") { + const legacyRefresh = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "refresh" }, + }); + if (legacyRefresh.status !== "ok") { + logger.warn("SessionManager: Legacy binding TTL refresh blocked", { + sessionId, + keyId, + reason: legacyRefresh.reason, + }); + } + } pipeline.setex( `session:${sessionId}:last_seen`, SessionManager.SESSION_TTL, @@ -609,6 +664,73 @@ export class SessionManager { } } + static getVersionedBindingCapabilityState(): VersionedBindingCapabilityState { + return readVersionedBindingCapabilityState(); + } + + static async getSessionBindingSnapshot( + sessionId: string, + keyId: number + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + } + + static async compareAndSetSessionProvider( + snapshot: SessionBindingSnapshot, + providerId: number + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return compareAndSetSessionBinding({ + sessionId: snapshot.sessionId, + keyId: snapshot.keyId, + expectedGeneration: snapshot.generation, + providerId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + } + + static async clearVersionedSessionProvider( + snapshot: SessionBindingSnapshot, + expectedProviderId: number | null, + cooldownTtlSeconds: number = 0 + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return clearVersionedSessionBinding({ + sessionId: snapshot.sessionId, + keyId: snapshot.keyId, + expectedGeneration: snapshot.generation, + expectedProviderId, + cooldownTtlSeconds, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + } + + static async isSessionProviderCoolingDown( + sessionId: string, + keyId: number, + providerId: number + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return readSessionProviderCooldown({ + sessionId, + keyId, + providerId, + redis, + }); + } + /** * 绑定 session 到 provider（TC-009 修复：使用 SET NX 避免竞态条件） */ @@ -621,35 +743,72 @@ export class SessionManager { if (!redis || redis.status !== "ready") return; try { - const key = `session:${sessionId}:provider`; - // 使用 SET ... NX 保证只有第一次绑定成功（原子操作） - const result = await redis.set( - key, - providerId.toString(), - "EX", - SessionManager.SESSION_TTL, - "NX" // Only set if not exists - ); - - if (result === "OK") { - if (keyId != null) { - await redis.setex( - `session:${sessionId}:key`, - SessionManager.SESSION_TTL, - keyId.toString() - ); - } - logger.trace("SessionManager: Bound session to provider", { + if (keyId != null) { + const binding = await readOrReconcileSessionBinding({ sessionId, - providerId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, }); - } else { - // 已绑定过，不覆盖（避免并发请求选择不同供应商） - logger.debug("SessionManager: Session already bound, skipping", { + if (binding.status === "ok") { + if (binding.snapshot.providerId !== null) { + logger.debug("SessionManager: Session already bound, skipping", { + sessionId, + attemptedProviderId: providerId, + }); + return; + } + + const updated = await compareAndSetSessionBinding({ + sessionId, + keyId, + expectedGeneration: binding.snapshot.generation, + providerId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (updated.status === "ok") { + logger.trace("SessionManager: Bound versioned session to provider", { + sessionId, + providerId, + }); + } + return; + } + if (!binding.legacyFallbackAllowed) { + logger.warn("SessionManager: Versioned session binding is not writable", { + sessionId, + keyId, + reason: binding.reason, + }); + return; + } + const legacy = await mutateLegacySessionBindingSafely({ sessionId, - attemptedProviderId: providerId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "bind_if_absent", providerId }, }); + if (legacy.status === "ok" && legacy.changed) { + logger.trace("SessionManager: Bound legacy session to provider", { + sessionId, + providerId, + }); + } else if (legacy.status !== "ok") { + logger.warn("SessionManager: Legacy session binding blocked", { + sessionId, + keyId, + reason: legacy.reason, + }); + } + return; } + + logger.warn("SessionManager: Cannot bind session without an API key owner", { + sessionId, + providerId, + }); } catch (error) { logger.error("SessionManager: Failed to bind provider", { error }); } @@ -667,6 +826,24 @@ export class SessionManager { try { if (keyId != null) { + const binding = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status === "ok") { + return binding.snapshot.providerId; + } + if (!binding.legacyFallbackAllowed) { + logger.warn("SessionManager: Versioned session binding is unavailable for reuse", { + sessionId, + keyId, + reason: binding.reason, + }); + return null; + } + const boundKeyId = await redis.get(`session:${sessionId}:key`); // Fail-closed：boundKeyId 缺失（TTL 漂移、旧绑定或写入路径未原子写 key）也视为校验失败， // 避免无法证明归属当前 key 的旧 provider binding 继续被复用。 @@ -699,41 +876,123 @@ export class SessionManager { */ static async clearSessionProvider( sessionId: string, - expectedProviderId?: number | null + expectedProviderId?: number | null, + keyId?: number | null ): Promise { const redis = getRedisClient(); if (!redis || redis.status !== "ready") return false; try { - const key = `session:${sessionId}:provider`; - const deleted = - expectedProviderId == null - ? await redis.del(key) - : Number( - await redis.eval( - ` - if redis.call("GET", KEYS[1]) == ARGV[1] then - return redis.call("DEL", KEYS[1]) - end - return 0 - `, - 1, - key, - expectedProviderId.toString() - ) - ); - logger.trace("SessionManager: Cleared session provider binding", { + if (keyId != null) { + const binding = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status === "ok") { + const currentProviderId = binding.snapshot.providerId; + if ( + currentProviderId === null || + (expectedProviderId != null && currentProviderId !== expectedProviderId) + ) { + return false; + } + + const cleared = await clearVersionedSessionBinding({ + sessionId, + keyId, + expectedGeneration: binding.snapshot.generation, + expectedProviderId: currentProviderId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + const didClear = cleared.status === "ok"; + logger.trace("SessionManager: Cleared versioned session provider binding", { + sessionId, + keyId, + expectedProviderId: expectedProviderId ?? null, + deleted: didClear, + }); + return didClear; + } + if (!binding.legacyFallbackAllowed) { + logger.warn("SessionManager: Versioned session binding clear blocked", { + sessionId, + keyId, + reason: binding.reason, + }); + return false; + } + const legacy = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "clear", expectedProviderId }, + }); + return legacy.status === "ok" && legacy.changed; + } + + logger.warn("SessionManager: Cannot clear session binding without an API key owner", { sessionId, expectedProviderId: expectedProviderId ?? null, - deleted: deleted > 0, }); - return deleted > 0; + return false; } catch (error) { logger.error("SessionManager: Failed to clear session provider", { error, sessionId }); return false; } } + static async clearSessionProviders( + sessionId: string, + expectedProviderIds: Iterable, + keyId?: number | null + ): Promise { + const providerIds = Array.from( + new Set( + Array.from(expectedProviderIds).filter( + (providerId) => Number.isSafeInteger(providerId) && providerId > 0 + ) + ) + ); + if (providerIds.length === 0 || keyId == null) return false; + + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return false; + + const binding = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status === "ok") { + const providerId = binding.snapshot.providerId; + if (providerId === null || !providerIds.includes(providerId)) return false; + const cleared = await clearVersionedSessionBinding({ + sessionId, + keyId, + expectedGeneration: binding.snapshot.generation, + expectedProviderId: providerId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + return cleared.status === "ok"; + } + if (!binding.legacyFallbackAllowed) return false; + + const legacy = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "clear", expectedProviderIds: providerIds }, + }); + return legacy.status === "ok" && legacy.changed; + } + /** * 获取当前绑定供应商的优先级 * @@ -743,19 +1002,16 @@ export class SessionManager { * @param sessionId - Session ID * @returns 优先级数字（数字越小优先级越高），如果未绑定或无法查询则返回 null */ - static async getSessionProviderPriority(sessionId: string): Promise { + static async getSessionProviderPriority( + sessionId: string, + keyId?: number | null + ): Promise { const redis = getRedisClient(); if (!redis || redis.status !== "ready") return null; try { - // 修复：从真实绑定关系读取（session:provider） - const providerIdStr = await redis.get(`session:${sessionId}:provider`); - if (!providerIdStr) { - return null; - } - - const providerId = parseInt(providerIdStr, 10); - if (Number.isNaN(providerId)) { + const providerId = await SessionManager.getSessionProvider(sessionId, keyId); + if (providerId === null) { return null; } @@ -797,26 +1053,91 @@ export class SessionManager { } try { - // ========== 情况 1：首次尝试成功 ========== - if (isFirstAttempt) { - const key = `session:${sessionId}:provider`; - // 使用 SET NX 绑定（避免覆盖并发请求） - const result = await redis.set( - key, - newProviderId.toString(), - "EX", - SessionManager.SESSION_TTL, - "NX" - ); + let versionedSnapshot: SessionBindingSnapshot | null = null; + let useLegacyBinding = false; + let legacyProviderId: number | null = null; - if (result === "OK") { - if (keyId != null) { - await redis.setex( - `session:${sessionId}:key`, - SessionManager.SESSION_TTL, - keyId.toString() - ); + if (keyId != null) { + const binding = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status === "ok") { + versionedSnapshot = binding.snapshot; + } else if (binding.legacyFallbackAllowed) { + const legacy = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "inspect" }, + }); + if (legacy.status !== "ok") { + return { + updated: false, + reason: "legacy_binding_conflict", + details: legacy.reason, + }; } + useLegacyBinding = true; + legacyProviderId = legacy.providerId; + } else { + return { + updated: false, + reason: "versioned_binding_conflict", + details: binding.reason, + }; + } + } else { + return { + updated: false, + reason: "binding_owner_unavailable", + details: "Cannot mutate a Session binding without an API key owner", + }; + } + + const persistBinding = async (onlyIfUnbound: boolean): Promise => { + if (versionedSnapshot) { + if (onlyIfUnbound && versionedSnapshot.providerId !== null) { + return false; + } + const result = await compareAndSetSessionBinding({ + sessionId, + keyId: versionedSnapshot.keyId, + expectedGeneration: versionedSnapshot.generation, + providerId: newProviderId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (result.status !== "ok") { + logger.warn("SessionManager: Versioned session binding CAS did not update", { + sessionId, + keyId: versionedSnapshot.keyId, + providerId: newProviderId, + reason: result.reason, + }); + return false; + } + return true; + } + + if (!useLegacyBinding) return false; + const result = await mutateLegacySessionBindingSafely({ + sessionId, + keyId: keyId!, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: onlyIfUnbound + ? { type: "bind_if_absent", providerId: newProviderId } + : { type: "set", providerId: newProviderId }, + }); + return result.status === "ok" && result.changed; + }; + + if (isFirstAttempt) { + if (await persistBinding(true)) { logger.info("SessionManager: Bound session to provider (first success)", { sessionId, providerId: newProviderId, @@ -827,31 +1148,23 @@ export class SessionManager { reason: "first_success", details: `首次成功，绑定到供应商 ${newProviderId} (priority=${newProviderPriority})`, }; - } else { - // 并发请求已经绑定了，放弃更新 - return { - updated: false, - reason: "concurrent_binding_exists", - details: "并发请求已绑定，跳过", - }; } + return { + updated: false, + reason: "concurrent_binding_exists", + details: "并发请求已绑定，跳过", + }; } - // ========== 情况 2：重试成功（需要智能决策）========== - - // 2.0 故障转移成功 或 竞速赢家强制改绑：无条件更新绑定 - // forceUpdate 在读取当前绑定/优先级/熔断状态之前短路，确保竞速赢家一定成为复用绑定。 if (isFailoverSuccess || forceUpdate) { - const pipeline = redis.pipeline(); - pipeline.setex( - `session:${sessionId}:provider`, - SessionManager.SESSION_TTL, - newProviderId.toString() - ); - if (keyId != null) { - pipeline.setex(`session:${sessionId}:key`, SessionManager.SESSION_TTL, keyId.toString()); + const updated = await persistBinding(false); + if (!updated) { + return { + updated: false, + reason: "concurrent_binding_changed", + details: "Session binding changed before the update committed", + }; } - await pipeline.exec(); const reason = isFailoverSuccess ? "failover_success" : "race_winner_forced"; logger.info( @@ -874,27 +1187,10 @@ export class SessionManager { }; } - // 2.1 获取当前绑定的供应商 ID - const currentProviderIdStr = await redis.get(`session:${sessionId}:provider`); - if (!currentProviderIdStr) { - // 没有绑定，使用 SET NX 绑定 - const key = `session:${sessionId}:provider`; - const result = await redis.set( - key, - newProviderId.toString(), - "EX", - SessionManager.SESSION_TTL, - "NX" - ); + const currentProviderId: number | null = versionedSnapshot?.providerId ?? legacyProviderId; - if (result === "OK") { - if (keyId != null) { - await redis.setex( - `session:${sessionId}:key`, - SessionManager.SESSION_TTL, - keyId.toString() - ); - } + if (currentProviderId === null) { + if (await persistBinding(true)) { logger.info("SessionManager: Bound session (no previous binding)", { sessionId, providerId: newProviderId, @@ -905,39 +1201,21 @@ export class SessionManager { reason: "no_previous_binding", details: `无绑定，绑定到供应商 ${newProviderId} (priority=${newProviderPriority})`, }; - } else { - return { - updated: false, - reason: "concurrent_binding_exists", - details: "并发请求已绑定", - }; } + return { + updated: false, + reason: "concurrent_binding_exists", + details: "并发请求已绑定", + }; } - const currentProviderId = parseInt(currentProviderIdStr, 10); - if (Number.isNaN(currentProviderId)) { - logger.warn("SessionManager: Invalid provider ID in Redis", { - currentProviderIdStr, - }); - return { updated: false, reason: "invalid_provider_id" }; - } - - // 2.2 查询当前供应商的详情（优先级 + 健康状态） const { findProviderById } = await import("@/repository/provider"); const currentProvider = await findProviderById(currentProviderId); if (!currentProvider) { - // 当前供应商不存在（可能被删除），直接更新 - const pipeline = redis.pipeline(); - pipeline.setex( - `session:${sessionId}:provider`, - SessionManager.SESSION_TTL, - newProviderId.toString() - ); - if (keyId != null) { - pipeline.setex(`session:${sessionId}:key`, SessionManager.SESSION_TTL, keyId.toString()); + if (!(await persistBinding(false))) { + return { updated: false, reason: "concurrent_binding_changed" }; } - await pipeline.exec(); logger.info("SessionManager: Updated binding (current provider not found)", { sessionId, @@ -955,20 +1233,10 @@ export class SessionManager { const currentPriority = currentProvider.priority || 0; - // 2.3 智能决策：优先级比较 + 健康检查 - - // ========== 规则 A：新供应商优先级更高（数字更小）→ 直接迁移 ========== if (newProviderPriority < currentPriority) { - const pipeline = redis.pipeline(); - pipeline.setex( - `session:${sessionId}:provider`, - SessionManager.SESSION_TTL, - newProviderId.toString() - ); - if (keyId != null) { - pipeline.setex(`session:${sessionId}:key`, SessionManager.SESSION_TTL, keyId.toString()); + if (!(await persistBinding(false))) { + return { updated: false, reason: "concurrent_binding_changed" }; } - await pipeline.exec(); logger.info("SessionManager: Migrated to higher priority provider", { sessionId, @@ -986,22 +1254,13 @@ export class SessionManager { }; } - // ========== 规则 B：新供应商优先级相同或更低 → 检查原供应商健康状态 ========== const { isCircuitOpen } = await import("@/lib/circuit-breaker"); const isCurrentCircuitOpen = await isCircuitOpen(currentProviderId); if (isCurrentCircuitOpen) { - // 原供应商已熔断 → 更新到新供应商（备用供应商接管） - const pipeline = redis.pipeline(); - pipeline.setex( - `session:${sessionId}:provider`, - SessionManager.SESSION_TTL, - newProviderId.toString() - ); - if (keyId != null) { - pipeline.setex(`session:${sessionId}:key`, SessionManager.SESSION_TTL, keyId.toString()); + if (!(await persistBinding(false))) { + return { updated: false, reason: "concurrent_binding_changed" }; } - await pipeline.exec(); logger.info("SessionManager: Migrated to backup provider (circuit open)", { sessionId, @@ -1019,7 +1278,6 @@ export class SessionManager { }; } - // 原供应商健康 + 优先级更高/相同 → 保持原绑定（尽量使用主供应商） logger.debug("SessionManager: Keeping current provider (healthy and higher/equal priority)", { sessionId, currentProviderId, @@ -2362,52 +2620,91 @@ export class SessionManager { // 使用 prompt_cache_key 作为新的 Session ID（添加前缀以区分） const codexSessionId = `codex_${promptCacheKey}`; - // 检查是否已经存在绑定 - const existingProvider = await redis.get(`session:${codexSessionId}:provider`); + if (keyId != null) { + const binding = await readOrReconcileSessionBinding({ + sessionId: codexSessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status === "ok") { + if (binding.snapshot.providerId !== null) { + logger.debug("SessionManager: Refreshed versioned Codex session TTL", { + sessionId: codexSessionId, + providerId: binding.snapshot.providerId, + }); + return { sessionId: codexSessionId, updated: false }; + } - if (existingProvider) { - // 已存在绑定，刷新 TTL - const pipeline = redis.pipeline(); - pipeline.expire(`session:${codexSessionId}:provider`, SessionManager.SESSION_TTL); - if (keyId != null) { - pipeline.setex( - `session:${codexSessionId}:key`, - SessionManager.SESSION_TTL, - keyId.toString() - ); + const updated = await compareAndSetSessionBinding({ + sessionId: codexSessionId, + keyId, + expectedGeneration: binding.snapshot.generation, + providerId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (updated.status === "ok") { + logger.info("SessionManager: Created versioned Codex session", { + sessionId: codexSessionId, + providerId, + }); + return { sessionId: codexSessionId, updated: true }; + } + return { sessionId: currentSessionId, updated: false }; + } + if (!binding.legacyFallbackAllowed) { + logger.warn("SessionManager: Codex session binding owner could not be verified", { + sessionId: codexSessionId, + keyId, + reason: binding.reason, + }); + return { sessionId: currentSessionId, updated: false }; } - await pipeline.exec(); - logger.debug("SessionManager: Refreshed Codex session TTL", { + const legacy = await mutateLegacySessionBindingSafely({ sessionId: codexSessionId, - providerId: parseInt(existingProvider, 10), + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "inspect" }, }); - return { sessionId: codexSessionId, updated: false }; - } + if (legacy.status !== "ok") { + logger.warn("SessionManager: Legacy Codex binding owner could not be verified", { + sessionId: codexSessionId, + keyId, + reason: legacy.reason, + }); + return { sessionId: currentSessionId, updated: false }; + } - // 新建绑定 - const pipeline = redis.pipeline(); - pipeline.setex( - `session:${codexSessionId}:provider`, - SessionManager.SESSION_TTL, - providerId.toString() - ); - if (keyId != null) { - pipeline.setex( - `session:${codexSessionId}:key`, - SessionManager.SESSION_TTL, - keyId.toString() - ); + if (legacy.providerId !== null) { + await mutateLegacySessionBindingSafely({ + sessionId: codexSessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "refresh" }, + }); + return { sessionId: codexSessionId, updated: false }; + } + + const bound = await mutateLegacySessionBindingSafely({ + sessionId: codexSessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "bind_if_absent", providerId }, + }); + if (bound.status === "ok") { + return { sessionId: codexSessionId, updated: bound.changed }; + } + return { sessionId: currentSessionId, updated: false }; } - await pipeline.exec(); - logger.info("SessionManager: Created Codex session from prompt_cache_key", { + logger.warn("SessionManager: Cannot bind Codex session without an API key owner", { sessionId: codexSessionId, - promptCacheKey, - providerId, - ttl: SessionManager.SESSION_TTL, }); - - return { sessionId: codexSessionId, updated: true }; + return { sessionId: currentSessionId, updated: false }; } catch (error) { logger.error("SessionManager: Failed to update Codex session", { error }); return { sessionId: currentSessionId, updated: false }; @@ -2423,7 +2720,10 @@ export class SessionManager { * @param sessionId - Session ID * @returns 是否成功删除 */ - static async terminateSession(sessionId: string): Promise { + static async terminateSession( + sessionId: string, + expectedProviderIds?: readonly number[] + ): Promise { const redis = getRedisClient(); if (!redis || redis.status !== "ready") { logger.warn("SessionManager: Redis not ready, cannot terminate session"); @@ -2435,6 +2735,7 @@ export class SessionManager { let providerId: number | null = null; let keyId: number | null = null; let userId: number | null = null; + let bindingTerminated = false; try { const [providerIdStr, keyIdStr, userIdStr] = await Promise.all([ @@ -2443,11 +2744,17 @@ export class SessionManager { redis.hget(`session:${sessionId}:info`, "userId"), ]); - providerId = providerIdStr ? parseInt(providerIdStr, 10) : null; - keyId = keyIdStr ? parseInt(keyIdStr, 10) : null; - userId = userIdStr ? parseInt(userIdStr, 10) : null; + providerId = providerIdStr ? Number(providerIdStr) : null; + keyId = keyIdStr ? Number(keyIdStr) : null; + userId = userIdStr ? Number(userIdStr) : null; - if (!Number.isFinite(userId)) { + if (providerId !== null && (!Number.isSafeInteger(providerId) || providerId <= 0)) { + providerId = null; + } + if (keyId !== null && (!Number.isSafeInteger(keyId) || keyId <= 0)) { + keyId = null; + } + if (userId !== null && (!Number.isSafeInteger(userId) || userId <= 0)) { userId = null; } } catch (lookupError) { @@ -2461,12 +2768,85 @@ export class SessionManager { ); } + if (keyId !== null) { + const binding = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status === "ok") { + providerId = binding.snapshot.providerId ?? providerId; + if ( + expectedProviderIds && + (binding.snapshot.providerId === null || + !expectedProviderIds.includes(binding.snapshot.providerId)) + ) { + return false; + } + + const terminated = await terminateVersionedSessionBinding({ + sessionId, + keyId, + expectedProviderId: expectedProviderIds + ? (binding.snapshot.providerId ?? undefined) + : undefined, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (terminated.status !== "ok") { + const latest = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + logger.warn("SessionManager: Versioned session termination blocked", { + sessionId, + keyId, + reason: terminated.reason, + latestStatus: latest.status, + }); + return false; + } + bindingTerminated = true; + } else if (binding.status === "unavailable" && binding.legacyFallbackAllowed) { + const legacy = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "terminate", expectedProviderIds }, + }); + if (legacy.status !== "ok") { + logger.warn("SessionManager: Legacy session termination blocked", { + sessionId, + keyId, + reason: legacy.reason, + }); + return false; + } + bindingTerminated = true; + } else { + logger.warn("SessionManager: Session binding termination blocked", { + sessionId, + keyId, + reason: binding.reason, + }); + return false; + } + } else if (providerId !== null || expectedProviderIds) { + logger.warn("SessionManager: Session binding owner unavailable during termination", { + sessionId, + providerId, + }); + return false; + } + // 2. 删除所有 Session 相关的 key const pipeline = redis.pipeline(); - // 基础绑定信息 - pipeline.del(`session:${sessionId}:provider`); - pipeline.del(`session:${sessionId}:key`); + // Binding mirrors are mutated only by the tenant-authorized helpers above. pipeline.del(`session:${sessionId}:info`); pipeline.del(`session:${sessionId}:last_seen`); pipeline.del(`session:${sessionId}:concurrent_count`); @@ -2514,7 +2894,7 @@ export class SessionManager { deletedKeys, }); - return deletedKeys > 0; + return bindingTerminated || deletedKeys > 0; } catch (error) { logger.error("SessionManager: Failed to terminate session", { error, @@ -2572,7 +2952,10 @@ export class SessionManager { return 0; } - const terminatedCount = await SessionManager.terminateSessionsBatch([...sessionIds]); + const terminatedCount = await SessionManager.terminateSessionsBatch( + [...sessionIds], + uniqueProviderIds + ); logger.info("SessionManager: Terminated provider sessions batch", { providerIds: uniqueProviderIds, sessionCount: sessionIds.size, @@ -2617,7 +3000,10 @@ export class SessionManager { * @param sessionIds - Session ID 列表 * @returns 成功终止的数量 */ - static async terminateSessionsBatch(sessionIds: string[]): Promise { + static async terminateSessionsBatch( + sessionIds: string[], + expectedProviderIds?: readonly number[] + ): Promise { if (sessionIds.length === 0) { return 0; } @@ -2637,7 +3023,7 @@ export class SessionManager { const chunk = sessionIds.slice(i, i + CHUNK_SIZE); const results = await Promise.all( chunk.map(async (sessionId) => { - const success = await SessionManager.terminateSession(sessionId); + const success = await SessionManager.terminateSession(sessionId, expectedProviderIds); return success ? 1 : 0; }) ); diff --git a/src/lib/session-tracker.ts b/src/lib/session-tracker.ts index dd278a521..878d0fc98 100644 --- a/src/lib/session-tracker.ts +++ b/src/lib/session-tracker.ts @@ -5,6 +5,10 @@ import { getUserActiveSessionsKey, } from "@/lib/redis/active-session-keys"; import { getRedisClient } from "./redis"; +import { + getVersionedBindingCapabilityState, + mutateLegacySessionBindingSafely, +} from "./redis/session-binding"; const PROVIDER_ACTIVE_SESSIONS_PATTERN = /^provider:(\d+):active_sessions$/; @@ -197,8 +201,24 @@ export class SessionTracker { try { const now = Date.now(); - const pipeline = redis.pipeline(); const ttlSeconds = SessionTracker.SESSION_TTL_SECONDS; + if (getVersionedBindingCapabilityState() === "unavailable") { + const legacyRefresh = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds, + redis, + mutation: { type: "refresh" }, + }); + if (legacyRefresh.status !== "ok") { + logger.warn("SessionTracker: Legacy binding TTL refresh blocked", { + sessionId, + keyId, + reason: legacyRefresh.reason, + }); + } + } + const pipeline = redis.pipeline(); const providerZSetKey = `provider:${providerId}:active_sessions`; const providerRefKey = `provider:${providerId}:active_session_refs`; const globalKey = getGlobalActiveSessionsKey(); @@ -222,10 +242,6 @@ export class SessionTracker { commandIndex++; } - pipeline.expire(`session:${sessionId}:provider`, ttlSeconds); - commandIndex++; - pipeline.expire(`session:${sessionId}:key`, ttlSeconds); - commandIndex++; pipeline.setex(`session:${sessionId}:last_seen`, ttlSeconds, now.toString()); commandIndex++; diff --git a/tests/configs/integration.config.ts b/tests/configs/integration.config.ts index ca2152353..58b4c0de3 100644 --- a/tests/configs/integration.config.ts +++ b/tests/configs/integration.config.ts @@ -10,6 +10,7 @@ export default createTestRunnerConfig({ "tests/integration/my-usage-imported-ledger.test.ts", "tests/integration/rolling-cost-redis.test.ts", "tests/integration/lease-settlement-redis.test.ts", + "tests/integration/session-binding-versioning-redis.test.ts", "tests/integration/db-pool-isolation-postgres.test.ts", "tests/integration/db-pool-slow-close-postgres.test.ts", "tests/integration/message-write-buffer-recovery-postgres.test.ts", diff --git a/tests/configs/session-binding.config.ts b/tests/configs/session-binding.config.ts new file mode 100644 index 000000000..0815cba93 --- /dev/null +++ b/tests/configs/session-binding.config.ts @@ -0,0 +1,9 @@ +import { createCoverageConfig } from "../vitest.base"; + +export default createCoverageConfig({ + name: "session-binding", + environment: "node", + testFiles: ["tests/unit/lib/redis/session-binding.test.ts"], + sourceFiles: ["src/lib/redis/session-binding.ts"], + thresholds: { lines: 80, functions: 80, branches: 75, statements: 80 }, +}); diff --git a/tests/integration/session-binding-versioning-redis.test.ts b/tests/integration/session-binding-versioning-redis.test.ts new file mode 100644 index 000000000..2520d9e98 --- /dev/null +++ b/tests/integration/session-binding-versioning-redis.test.ts @@ -0,0 +1,503 @@ +import { randomUUID } from "node:crypto"; +import Redis from "ioredis"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "vitest"; +import { + buildCanonicalSessionBindingKey, + buildLegacySessionOwnerKey, + buildLegacySessionProviderKey, + buildSessionBindingKeys, + buildSessionProviderCooldownKey, + clearSessionBinding, + compareAndSetSessionBinding, + ensureVersionedBindingCapability, + isSessionProviderCoolingDown, + readOrReconcileSessionBinding, + resetVersionedBindingCapabilityForTests, + type SessionBindingOkResult, + type SessionBindingResult, +} from "@/lib/redis/session-binding"; + +const HAS_REDIS = Boolean(process.env.REDIS_URL); +const EXPECTED_CAPABILITY_RAW = process.env.EXPECT_VERSIONED_BINDING_CAPABILITY; + +if ( + EXPECTED_CAPABILITY_RAW !== undefined && + EXPECTED_CAPABILITY_RAW !== "available" && + EXPECTED_CAPABILITY_RAW !== "unavailable" +) { + throw new Error("EXPECT_VERSIONED_BINDING_CAPABILITY must be either available or unavailable"); +} + +const EXPECTED_CAPABILITY = EXPECTED_CAPABILITY_RAW ?? "available"; +const runWithRedis = describe.skipIf(!HAS_REDIS); +const runWithVersionedBinding = describe.skipIf( + !HAS_REDIS || EXPECTED_CAPABILITY === "unavailable" +); +const TEST_PREFIX = `it-session-binding-${Date.now()}-${randomUUID()}`; +const BINDING_TTL_SECONDS = 90; +const COOLDOWN_TTL_SECONDS = 45; + +let redis: Redis; +let sequence = 0; +const touchedKeys = new Set(); + +function nextSessionId(label: string): string { + sequence += 1; + return `${TEST_PREFIX}:${label}:${sequence}`; +} + +function rememberBindingKeys( + sessionId: string, + keyIds: number[], + cooldownProviders: number[] = [] +): void { + for (const keyId of keyIds) { + const keys = buildSessionBindingKeys(sessionId, keyId); + touchedKeys.add(keys.canonical); + touchedKeys.add(keys.legacyProvider); + touchedKeys.add(keys.legacyOwner); + for (const providerId of cooldownProviders) { + touchedKeys.add(buildSessionProviderCooldownKey(sessionId, keyId, providerId)); + } + } +} + +async function deleteKeysIndividually(keys: Iterable): Promise { + for (const key of keys) { + await redis.del(key); + } +} + +async function scanKeys(pattern: string): Promise { + let cursor = "0"; + const keys: string[] = []; + do { + const [nextCursor, page] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 100); + cursor = nextCursor; + keys.push(...page); + } while (cursor !== "0"); + return keys.sort(); +} + +async function cleanupTouchedKeys(): Promise { + await deleteKeysIndividually(touchedKeys); + touchedKeys.clear(); + + // Probe keys use a reserved isolated namespace and may remain only when a + // cluster rejects the probe's multi-key cleanup with CROSSSLOT. + const probeKeys = await scanKeys("session-binding-capability-probe:*"); + await deleteKeysIndividually(probeKeys); +} + +function requireOk(result: SessionBindingResult): SessionBindingOkResult { + if (result.status !== "ok") { + throw new Error(`Expected successful session binding result, got ${JSON.stringify(result)}`); + } + return result; +} + +async function readBinding(sessionId: string, keyId: number, ttlSeconds = BINDING_TTL_SECONDS) { + rememberBindingKeys(sessionId, [keyId]); + return readOrReconcileSessionBinding({ sessionId, keyId, ttlSeconds, redis }); +} + +async function bindProvider( + sessionId: string, + keyId: number, + expectedGeneration: string, + providerId: number +) { + rememberBindingKeys(sessionId, [keyId]); + return compareAndSetSessionBinding({ + sessionId, + keyId, + expectedGeneration, + providerId, + ttlSeconds: BINDING_TTL_SECONDS, + redis, + }); +} + +beforeAll(async () => { + if (!HAS_REDIS) return; + redis = new Redis(process.env.REDIS_URL!, { + lazyConnect: true, + enableOfflineQueue: false, + maxRetriesPerRequest: 1, + }); + await redis.connect(); + await expect(redis.ping()).resolves.toBe("PONG"); +}); + +beforeEach(() => { + resetVersionedBindingCapabilityForTests(); +}); + +afterEach(async () => { + if (HAS_REDIS) { + await cleanupTouchedKeys(); + } + resetVersionedBindingCapabilityForTests(); +}); + +afterAll(async () => { + if (HAS_REDIS) { + await cleanupTouchedKeys(); + if (redis.status !== "end") { + await redis.quit(); + } + } + resetVersionedBindingCapabilityForTests(); +}); + +runWithRedis("versioned session binding Redis capability", () => { + test("matches the explicitly expected capability and cleans isolated probe keys", async () => { + const probeKeysBefore = await scanKeys("session-binding-capability-probe:*"); + + const capability = await ensureVersionedBindingCapability(redis); + + expect(capability).toBe(EXPECTED_CAPABILITY); + if (capability === "available") { + expect(await scanKeys("session-binding-capability-probe:*")).toEqual(probeKeysBefore); + } + }); + + test.runIf(EXPECTED_CAPABILITY === "unavailable")( + "fails closed without creating a business binding when capability is unavailable", + async () => { + const sessionId = nextSessionId("unavailable"); + const keyId = 9001; + rememberBindingKeys(sessionId, [keyId]); + + const result = await readBinding(sessionId, keyId); + + expect(result).toMatchObject({ + status: "unavailable", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + }); + expect(await redis.exists(buildCanonicalSessionBindingKey(sessionId, keyId))).toBe(0); + expect(await redis.exists(buildLegacySessionOwnerKey(sessionId))).toBe(0); + expect(await redis.exists(buildLegacySessionProviderKey(sessionId))).toBe(0); + } + ); +}); + +runWithVersionedBinding("versioned session binding reconcile", () => { + test("creates a true empty null tombstone and refreshes TTL without rotating generation", async () => { + const sessionId = nextSessionId("empty"); + const keyId = 1001; + const keys = buildSessionBindingKeys(sessionId, keyId); + + const created = requireOk(await readBinding(sessionId, keyId)); + expect(created).toMatchObject({ + source: "created", + snapshot: { sessionId, keyId, providerId: null }, + }); + expect(await redis.hget(keys.canonical, "key_id")).toBe(String(keyId)); + expect(await redis.hget(keys.canonical, "generation")).toBe(created.snapshot.generation); + expect(await redis.hget(keys.canonical, "provider_id")).toBeNull(); + expect(await redis.get(keys.legacyOwner)).toBe(String(keyId)); + expect(await redis.exists(keys.legacyProvider)).toBe(0); + + await redis.expire(keys.canonical, 5); + await redis.expire(keys.legacyOwner, 5); + + const reread = requireOk(await readBinding(sessionId, keyId)); + expect(reread.source).toBe("existing"); + expect(reread.snapshot.generation).toBe(created.snapshot.generation); + expect(reread.snapshot.providerId).toBeNull(); + expect(await redis.ttl(keys.canonical)).toBeGreaterThan(BINDING_TTL_SECONDS - 5); + expect(await redis.ttl(keys.legacyOwner)).toBeGreaterThan(BINDING_TTL_SECONDS - 5); + }); + + test("lazy-upgrades a matching legacy provider and owner", async () => { + const sessionId = nextSessionId("legacy-provider"); + const keyId = 1002; + const providerId = 2002; + const keys = buildSessionBindingKeys(sessionId, keyId); + rememberBindingKeys(sessionId, [keyId]); + await redis.setex(keys.legacyOwner, BINDING_TTL_SECONDS, String(keyId)); + await redis.setex(keys.legacyProvider, BINDING_TTL_SECONDS, String(providerId)); + + const upgraded = requireOk(await readBinding(sessionId, keyId)); + + expect(upgraded).toMatchObject({ + source: "legacy_upgraded", + snapshot: { sessionId, keyId, providerId }, + }); + expect(await redis.hgetall(keys.canonical)).toMatchObject({ + key_id: String(keyId), + generation: upgraded.snapshot.generation, + provider_id: String(providerId), + }); + }); + + test("lazy-upgrades a matching owner with no legacy provider as a null tombstone", async () => { + const sessionId = nextSessionId("legacy-null"); + const keyId = 1003; + const keys = buildSessionBindingKeys(sessionId, keyId); + rememberBindingKeys(sessionId, [keyId]); + await redis.setex(keys.legacyOwner, BINDING_TTL_SECONDS, String(keyId)); + + const upgraded = requireOk(await readBinding(sessionId, keyId)); + + expect(upgraded.source).toBe("legacy_upgraded"); + expect(upgraded.snapshot.providerId).toBeNull(); + expect(await redis.hget(keys.canonical, "provider_id")).toBeNull(); + expect(await redis.exists(keys.legacyProvider)).toBe(0); + }); + + test("rejects a foreign legacy owner without importing or overwriting it", async () => { + const sessionId = nextSessionId("foreign-owner"); + const keyId = 1004; + const foreignKeyId = 7777; + const providerId = 2004; + const keys = buildSessionBindingKeys(sessionId, keyId); + rememberBindingKeys(sessionId, [keyId]); + await redis.setex(keys.legacyOwner, BINDING_TTL_SECONDS, String(foreignKeyId)); + await redis.setex(keys.legacyProvider, BINDING_TTL_SECONDS, String(providerId)); + + const result = await readBinding(sessionId, keyId); + + expect(result).toEqual({ + status: "conflict", + reason: "foreign_legacy_owner", + legacyFallbackAllowed: false, + }); + expect(await redis.exists(keys.canonical)).toBe(0); + expect(await redis.get(keys.legacyOwner)).toBe(String(foreignKeyId)); + expect(await redis.get(keys.legacyProvider)).toBe(String(providerId)); + }); + + test("rejects an orphan legacy provider without claiming ownership", async () => { + const sessionId = nextSessionId("orphan-provider"); + const keyId = 1005; + const providerId = 2005; + const keys = buildSessionBindingKeys(sessionId, keyId); + rememberBindingKeys(sessionId, [keyId]); + await redis.setex(keys.legacyProvider, BINDING_TTL_SECONDS, String(providerId)); + + const result = await readBinding(sessionId, keyId); + + expect(result).toEqual({ + status: "conflict", + reason: "orphan_legacy_provider", + legacyFallbackAllowed: false, + }); + expect(await redis.exists(keys.canonical)).toBe(0); + expect(await redis.exists(keys.legacyOwner)).toBe(0); + expect(await redis.get(keys.legacyProvider)).toBe(String(providerId)); + }); + + test("rejects a non-positive legacy provider without creating canonical state", async () => { + const sessionId = nextSessionId("invalid-provider"); + const keyId = 1006; + const keys = buildSessionBindingKeys(sessionId, keyId); + rememberBindingKeys(sessionId, [keyId]); + await redis.setex(keys.legacyOwner, BINDING_TTL_SECONDS, String(keyId)); + await redis.setex(keys.legacyProvider, BINDING_TTL_SECONDS, "-1"); + + const result = await readBinding(sessionId, keyId); + + expect(result).toEqual({ + status: "conflict", + reason: "invalid_legacy_provider", + legacyFallbackAllowed: false, + }); + expect(await redis.exists(keys.canonical)).toBe(0); + }); + + test("fails closed for missing and contradictory mirrors without repairing either side", async () => { + const sessionId = nextSessionId("mirror-conflict"); + const keyId = 1006; + const providerId = 2006; + const keys = buildSessionBindingKeys(sessionId, keyId); + const created = requireOk(await readBinding(sessionId, keyId)); + const bound = requireOk( + await bindProvider(sessionId, keyId, created.snapshot.generation, providerId) + ); + + await redis.del(keys.legacyOwner); + const missing = await readBinding(sessionId, keyId); + expect(missing).toEqual({ + status: "conflict", + reason: "mirror_missing", + legacyFallbackAllowed: false, + }); + expect(await redis.hget(keys.canonical, "generation")).toBe(bound.snapshot.generation); + + await redis.setex(keys.legacyOwner, BINDING_TTL_SECONDS, String(keyId)); + await redis.setex(keys.legacyProvider, BINDING_TTL_SECONDS, String(providerId + 1)); + const contradictory = await readBinding(sessionId, keyId); + expect(contradictory).toEqual({ + status: "conflict", + reason: "mirror_conflict", + legacyFallbackAllowed: false, + }); + expect(await redis.hget(keys.canonical, "provider_id")).toBe(String(providerId)); + expect(await redis.get(keys.legacyProvider)).toBe(String(providerId + 1)); + }); + + test("allows only one tenant to initialize the same empty legacy session", async () => { + const sessionId = nextSessionId("tenant-race"); + const keyA = 1101; + const keyB = 1102; + rememberBindingKeys(sessionId, [keyA, keyB]); + + const [resultA, resultB] = await Promise.all([ + readOrReconcileSessionBinding({ + sessionId, + keyId: keyA, + ttlSeconds: BINDING_TTL_SECONDS, + redis, + }), + readOrReconcileSessionBinding({ + sessionId, + keyId: keyB, + ttlSeconds: BINDING_TTL_SECONDS, + redis, + }), + ]); + + const winner = resultA.status === "ok" ? resultA : resultB.status === "ok" ? resultB : null; + const loser = resultA.status === "conflict" ? resultA : resultB; + expect(winner).not.toBeNull(); + expect(loser).toMatchObject({ status: "conflict", reason: "foreign_legacy_owner" }); + if (!winner) throw new Error("Expected one tenant to win initialization"); + + const losingKeyId = winner.snapshot.keyId === keyA ? keyB : keyA; + expect(await redis.get(buildLegacySessionOwnerKey(sessionId))).toBe( + String(winner.snapshot.keyId) + ); + expect( + await redis.exists(buildCanonicalSessionBindingKey(sessionId, winner.snapshot.keyId)) + ).toBe(1); + expect(await redis.exists(buildCanonicalSessionBindingKey(sessionId, losingKeyId))).toBe(0); + }); +}); + +runWithVersionedBinding("versioned session binding mutation", () => { + test("rotates generation across CAS and rejects a stale ABA clear", async () => { + const sessionId = nextSessionId("aba"); + const keyId = 1201; + const providerP = 2201; + const providerQ = 2202; + rememberBindingKeys(sessionId, [keyId], [providerP]); + + const initial = requireOk(await readBinding(sessionId, keyId)); + const firstP = requireOk( + await bindProvider(sessionId, keyId, initial.snapshot.generation, providerP) + ); + + const staleCas = await bindProvider(sessionId, keyId, initial.snapshot.generation, providerQ); + expect(staleCas).toMatchObject({ status: "conflict", reason: "generation_mismatch" }); + + const boundQ = requireOk( + await bindProvider(sessionId, keyId, firstP.snapshot.generation, providerQ) + ); + const secondP = requireOk( + await bindProvider(sessionId, keyId, boundQ.snapshot.generation, providerP) + ); + expect( + new Set([ + initial.snapshot.generation, + firstP.snapshot.generation, + boundQ.snapshot.generation, + secondP.snapshot.generation, + ]).size + ).toBe(4); + + const staleClear = await clearSessionBinding({ + sessionId, + keyId, + expectedGeneration: firstP.snapshot.generation, + expectedProviderId: providerP, + ttlSeconds: BINDING_TTL_SECONDS, + cooldownTtlSeconds: COOLDOWN_TTL_SECONDS, + redis, + }); + expect(staleClear).toMatchObject({ status: "conflict", reason: "generation_mismatch" }); + expect(await redis.get(buildLegacySessionProviderKey(sessionId))).toBe(String(providerP)); + expect(await redis.hget(buildCanonicalSessionBindingKey(sessionId, keyId), "generation")).toBe( + secondP.snapshot.generation + ); + expect(await redis.exists(buildSessionProviderCooldownKey(sessionId, keyId, providerP))).toBe( + 0 + ); + }); + + test("atomically clears a provider and writes a tenant-scoped cooldown", async () => { + const sessionId = nextSessionId("clear-cooldown"); + const keyId = 1202; + const otherKeyId = 1203; + const providerId = 2203; + rememberBindingKeys(sessionId, [keyId, otherKeyId], [providerId]); + const keys = buildSessionBindingKeys(sessionId, keyId); + const cooldownKey = buildSessionProviderCooldownKey(sessionId, keyId, providerId); + + const initial = requireOk(await readBinding(sessionId, keyId)); + const bound = requireOk( + await bindProvider(sessionId, keyId, initial.snapshot.generation, providerId) + ); + const cleared = requireOk( + await clearSessionBinding({ + sessionId, + keyId, + expectedGeneration: bound.snapshot.generation, + expectedProviderId: providerId, + ttlSeconds: BINDING_TTL_SECONDS, + cooldownTtlSeconds: COOLDOWN_TTL_SECONDS, + redis, + }) + ); + + expect(cleared.source).toBe("cleared"); + expect(cleared.snapshot.providerId).toBeNull(); + expect(cleared.snapshot.generation).not.toBe(bound.snapshot.generation); + expect(await redis.hget(keys.canonical, "provider_id")).toBeNull(); + expect(await redis.get(keys.legacyOwner)).toBe(String(keyId)); + expect(await redis.exists(keys.legacyProvider)).toBe(0); + expect(await redis.get(cooldownKey)).toBe(cleared.snapshot.generation); + expect(await redis.ttl(cooldownKey)).toBeGreaterThan(COOLDOWN_TTL_SECONDS - 5); + + await expect( + isSessionProviderCoolingDown({ sessionId, keyId, providerId, redis }) + ).resolves.toEqual({ status: "ok", coolingDown: true, legacyFallbackAllowed: false }); + await expect( + isSessionProviderCoolingDown({ sessionId, keyId: otherKeyId, providerId, redis }) + ).resolves.toEqual({ status: "ok", coolingDown: false, legacyFallbackAllowed: false }); + }); + + test("does not initialize CAS state after canonical expiry", async () => { + const sessionId = nextSessionId("canonical-missing"); + const keyId = 1204; + const providerId = 2204; + const keys = buildSessionBindingKeys(sessionId, keyId); + const initial = requireOk(await readBinding(sessionId, keyId)); + await redis.del(keys.canonical); + + const result = await bindProvider(sessionId, keyId, initial.snapshot.generation, providerId); + + expect(result).toMatchObject({ status: "conflict", reason: "canonical_missing" }); + expect(await redis.exists(keys.canonical)).toBe(0); + expect(await redis.get(keys.legacyOwner)).toBe(String(keyId)); + expect(await redis.exists(keys.legacyProvider)).toBe(0); + }); + + test("fails closed when canonical generation is missing", async () => { + const sessionId = nextSessionId("generation-missing"); + const keyId = 1205; + const providerId = 2205; + const keys = buildSessionBindingKeys(sessionId, keyId); + const initial = requireOk(await readBinding(sessionId, keyId)); + await redis.hdel(keys.canonical, "generation"); + + const readResult = await readBinding(sessionId, keyId); + expect(readResult).toMatchObject({ status: "conflict", reason: "canonical_corrupt" }); + + const casResult = await bindProvider(sessionId, keyId, initial.snapshot.generation, providerId); + expect(casResult).toMatchObject({ status: "conflict", reason: "canonical_corrupt" }); + expect(await redis.exists(keys.legacyProvider)).toBe(0); + }); +}); diff --git a/tests/unit/lib/redis/client.test.ts b/tests/unit/lib/redis/client.test.ts index c6dfa6c49..4553739d5 100644 --- a/tests/unit/lib/redis/client.test.ts +++ b/tests/unit/lib/redis/client.test.ts @@ -116,6 +116,16 @@ describe("getRedisClient", () => { expect(mocks.MockRedis).toHaveBeenCalledTimes(1); }); + it("replaces the singleton when REDIS_URL changes", () => { + getRedisClient({ allowWhenRateLimitDisabled: true }); + process.env.REDIS_URL = "redis://localhost:6380"; + + getRedisClient({ allowWhenRateLimitDisabled: true }); + + expect(mocks.mockDisconnect).toHaveBeenCalledTimes(1); + expect(mocks.MockRedis).toHaveBeenCalledTimes(2); + }); + it("creates new client when existing singleton has status=end", () => { getRedisClient({ allowWhenRateLimitDisabled: true }); mocks.state.status = "end"; diff --git a/tests/unit/lib/redis/session-binding.test.ts b/tests/unit/lib/redis/session-binding.test.ts new file mode 100644 index 000000000..739ac07b3 --- /dev/null +++ b/tests/unit/lib/redis/session-binding.test.ts @@ -0,0 +1,907 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("server-only", () => ({})); +vi.mock("@/lib/logger", () => ({ + logger: { debug: vi.fn(), error: vi.fn(), info: vi.fn(), warn: vi.fn() }, +})); + +import { + buildCanonicalSessionBindingKey, + buildLegacySessionOwnerKey, + buildLegacySessionProviderKey, + buildSessionBindingKeys, + buildSessionProviderCooldownKey, + clearSessionBinding, + compareAndSetSessionBinding, + ensureVersionedBindingCapability, + getVersionedBindingCapabilityState, + isSessionProviderCoolingDown, + mutateLegacySessionBindingSafely, + readOrReconcileSessionBinding, + refreshSessionBinding, + resetVersionedBindingCapabilityForTests, + terminateSessionBinding, + type SessionBindingRedisClient, +} from "@/lib/redis/session-binding"; +import { + CAS_SESSION_BINDING, + CLEAR_SESSION_BINDING, + READ_OR_RECONCILE_SESSION_BINDING, + TERMINATE_SESSION_BINDING, +} from "@/lib/redis/lua-scripts"; + +type EvalResponse = unknown | Error | ((args: unknown[]) => unknown | Promise); + +interface MockRedisOptions { + cleanupFails?: boolean; + evalSha?: boolean; + evalShaNoScriptOnce?: boolean; + operationResponses?: Partial>; + probeFails?: boolean; + probeGate?: Promise; + status?: string; + cooldownValue?: string | null; +} + +function createMockRedis(options: MockRedisOptions = {}) { + const listeners = new Map void>>(); + const probeCooldowns = new Map(); + let probeFails = options.probeFails ?? false; + let cleanupFails = options.cleanupFails ?? false; + let status = options.status ?? "ready"; + + const evalMock = vi.fn(async (...args: unknown[]) => { + const [script, numberOfKeys] = args as [string, number]; + const firstKey = String(args[2]); + const isProbe = firstKey.startsWith("session-binding-capability-probe:"); + + if (isProbe) { + await options.probeGate; + if (probeFails) throw new Error("CROSSSLOT keys in request do not hash to the same slot"); + if (script === READ_OR_RECONCILE_SESSION_BINDING) { + return ["ok", "created", String(args[6]), ""]; + } + if (script === CAS_SESSION_BINDING) { + return ["ok", "updated", String(args[7]), String(args[8])]; + } + if (script === CLEAR_SESSION_BINDING) { + probeCooldowns.set(String(args[5]), String(args[8])); + return ["ok", "cleared", String(args[8]), ""]; + } + throw new Error("Unexpected probe script"); + } + + const queue = options.operationResponses?.[script]; + const response = queue?.shift(); + if (response instanceof Error) throw response; + if (typeof response === "function") return response(args); + if (response !== undefined) return response; + throw new Error(`Missing operation response for ${numberOfKeys} key script`); + }); + + const getMock = vi.fn(async (key: string) => { + if (probeCooldowns.has(key)) return probeCooldowns.get(key) ?? null; + return options.cooldownValue ?? null; + }); + const delMock = vi.fn(async (..._keys: string[]) => { + if (cleanupFails) throw new Error("cleanup failed"); + return 4; + }); + const existsMock = vi.fn(async () => 0); + const expireMock = vi.fn(async () => 1); + const setMock = vi.fn(async () => "OK"); + const setexMock = vi.fn(async () => "OK"); + const evalShaMock = vi.fn(async (..._args: unknown[]) => { + if (options.evalShaNoScriptOnce) { + options.evalShaNoScriptOnce = false; + throw new Error("NOSCRIPT No matching script"); + } + return ["ok", "existing", "generation-sha", "8"]; + }); + const onMock = vi.fn((event: string, listener: (...args: unknown[]) => void) => { + const callbacks = listeners.get(event) ?? new Set(); + callbacks.add(listener); + listeners.set(event, callbacks); + return redis; + }); + const offMock = vi.fn((event: string, listener: (...args: unknown[]) => void) => { + listeners.get(event)?.delete(listener); + return redis; + }); + + const redis = { + get status() { + return status; + }, + eval: evalMock, + ...(options.evalSha ? { evalsha: evalShaMock } : {}), + get: getMock, + del: delMock, + exists: existsMock, + expire: expireMock, + on: onMock, + off: offMock, + set: setMock, + setex: setexMock, + } satisfies SessionBindingRedisClient; + + return { + redis, + evalMock, + evalShaMock, + getMock, + delMock, + existsMock, + expireMock, + setMock, + setexMock, + onMock, + offMock, + emit(event: string) { + for (const listener of listeners.get(event) ?? []) listener(); + }, + setStatus(nextStatus: string) { + status = nextStatus; + }, + setProbeFails(value: boolean) { + probeFails = value; + }, + setCleanupFails(value: boolean) { + cleanupFails = value; + }, + }; +} + +describe("session binding key builders", () => { + it("scopes canonical and cooldown keys by API key while preserving legacy mirrors", () => { + const canonical = buildCanonicalSessionBindingKey("session:{a}", 17); + const cooldown = buildSessionProviderCooldownKey("session:{a}", 17, 4); + + expect(canonical).toMatch(/^session-binding:v1:\{[a-f0-9]{64}\}:binding$/); + expect(cooldown).toMatch(/^session-binding:v1:\{[a-f0-9]{64}\}:provider:4:cooldown$/); + expect(canonical.match(/\{([^}]+)\}/)?.[1]).toBe(cooldown.match(/\{([^}]+)\}/)?.[1]); + expect(buildCanonicalSessionBindingKey("session:a", 18)).not.toBe( + buildCanonicalSessionBindingKey("session:a", 17) + ); + expect(buildLegacySessionProviderKey("session:{a}")).toBe("session:session:{a}:provider"); + expect(buildLegacySessionOwnerKey("session:{a}")).toBe("session:session:{a}:key"); + }); + + it("supports an isolated namespace without changing the production key shape", () => { + const keys = buildSessionBindingKeys("sid", 9, "probe"); + expect(keys.canonical).toMatch(/^probe:session-binding:v1:\{[a-f0-9]{64}\}:binding$/); + expect(keys.legacyProvider).toBe("probe:session:sid:provider"); + expect(keys.legacyOwner).toBe("probe:session:sid:key"); + }); +}); + +describe("versioned binding capability", () => { + beforeEach(() => { + resetVersionedBindingCapabilityForTests(); + }); + + it("probes reconcile, CAS, clear, cooldown, and cleanup exactly once per connection", async () => { + const mock = createMockRedis(); + + await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("available"); + await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("available"); + + expect(getVersionedBindingCapabilityState()).toBe("available"); + expect(mock.evalMock).toHaveBeenCalledTimes(3); + expect(mock.getMock).toHaveBeenCalledTimes(1); + expect(mock.delMock).toHaveBeenCalledTimes(4); + expect(mock.delMock.mock.calls.every((call) => call.length === 1)).toBe(true); + expect(mock.onMock.mock.calls.map(([event]) => event)).toEqual([ + "close", + "connect", + "end", + "ready", + "reconnecting", + ]); + }); + + it("shares one in-flight capability probe across concurrent callers", async () => { + let releaseProbe: (() => void) | undefined; + const probeGate = new Promise((resolve) => { + releaseProbe = resolve; + }); + const mock = createMockRedis({ probeGate }); + + const first = ensureVersionedBindingCapability(mock.redis); + const second = ensureVersionedBindingCapability(mock.redis); + await vi.waitFor(() => expect(mock.evalMock).toHaveBeenCalledTimes(1)); + releaseProbe?.(); + + await expect(Promise.all([first, second])).resolves.toEqual(["available", "available"]); + expect(mock.evalMock).toHaveBeenCalledTimes(3); + }); + + it("stays unavailable on the same connection after a capability failure", async () => { + const mock = createMockRedis({ probeFails: true }); + + await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("unavailable"); + mock.setProbeFails(false); + await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("unavailable"); + + expect(mock.evalMock).toHaveBeenCalledTimes(1); + expect(getVersionedBindingCapabilityState()).toBe("unavailable"); + }); + + it("resets to unknown on reconnect and probes the new connection epoch", async () => { + const mock = createMockRedis({ probeFails: true }); + await ensureVersionedBindingCapability(mock.redis); + mock.setProbeFails(false); + + mock.emit("reconnecting"); + expect(getVersionedBindingCapabilityState()).toBe("unknown"); + await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("available"); + + expect(mock.evalMock).toHaveBeenCalledTimes(4); + }); + + it("automatically probes when a reconnected client becomes ready", async () => { + const mock = createMockRedis({ probeFails: true }); + await ensureVersionedBindingCapability(mock.redis); + mock.setProbeFails(false); + + mock.emit("close"); + expect(getVersionedBindingCapabilityState()).toBe("unknown"); + mock.emit("ready"); + + await vi.waitFor(() => expect(getVersionedBindingCapabilityState()).toBe("available")); + expect(mock.evalMock).toHaveBeenCalledTimes(4); + }); + + it("does not become available when isolated probe cleanup fails", async () => { + const mock = createMockRedis({ cleanupFails: true }); + + await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("unavailable"); + expect(mock.evalMock).toHaveBeenCalledTimes(3); + expect(mock.delMock).toHaveBeenCalledTimes(4); + }); + + it("detaches lifecycle listeners when tests reset state", async () => { + const mock = createMockRedis(); + await ensureVersionedBindingCapability(mock.redis); + + resetVersionedBindingCapabilityForTests(); + + expect(mock.offMock.mock.calls.map(([event]) => event)).toEqual([ + "close", + "connect", + "end", + "ready", + "reconnecting", + ]); + expect(getVersionedBindingCapabilityState()).toBe("unknown"); + }); + + it("reports unknown without creating a probe when Redis is not configured", async () => { + await expect(ensureVersionedBindingCapability()).resolves.toBe("unknown"); + expect(getVersionedBindingCapabilityState()).toBe("unknown"); + }); +}); + +describe("versioned session binding operations", () => { + beforeEach(() => { + resetVersionedBindingCapabilityForTests(); + }); + + it("reads a newly initialized null tombstone", async () => { + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [(args) => ["ok", "created", String(args[6]), ""]], + }, + }); + + const result = await readOrReconcileSessionBinding({ + sessionId: "fresh", + keyId: 7, + ttlSeconds: 90, + redis: mock.redis, + }); + + expect(result).toMatchObject({ + status: "ok", + source: "created", + snapshot: { sessionId: "fresh", keyId: 7, providerId: null }, + legacyFallbackAllowed: false, + }); + const operation = mock.evalMock.mock.calls.at(-1); + expect(operation?.slice(0, 5)).toEqual([ + READ_OR_RECONCILE_SESSION_BINDING, + 3, + buildCanonicalSessionBindingKey("fresh", 7), + "session:fresh:provider", + "session:fresh:key", + ]); + expect(operation?.at(-1)).toBe("90"); + }); + + it("parses an upgraded provider binding from Buffer values", async () => { + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [ + [ + Buffer.from("ok"), + Buffer.from("legacy_upgraded"), + Buffer.from("generation-a"), + Buffer.from("12"), + ], + ], + }, + }); + + const result = await refreshSessionBinding({ + sessionId: "legacy", + keyId: 3, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "ok", + source: "legacy_upgraded", + snapshot: { + sessionId: "legacy", + keyId: 3, + providerId: 12, + generation: "generation-a", + }, + legacyFallbackAllowed: false, + }); + }); + + it("returns tenant conflicts without disabling capability", async () => { + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [ + ["conflict", "foreign_legacy_owner"], + ["ok", "existing", "generation-b", "6"], + ], + }, + }); + + const first = await readOrReconcileSessionBinding({ + sessionId: "shared", + keyId: 2, + redis: mock.redis, + }); + const second = await readOrReconcileSessionBinding({ + sessionId: "shared", + keyId: 2, + redis: mock.redis, + }); + + expect(first).toEqual({ + status: "conflict", + reason: "foreign_legacy_owner", + legacyFallbackAllowed: false, + }); + expect(second.status).toBe("ok"); + expect(getVersionedBindingCapabilityState()).toBe("available"); + }); + + it("fails closed on unknown conflict reasons without disabling capability", async () => { + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [["conflict", "future_conflict_reason"]], + }, + }); + + const result = await readOrReconcileSessionBinding({ + sessionId: "shared", + keyId: 2, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "unknown_conflict", + legacyFallbackAllowed: false, + }); + expect(getVersionedBindingCapabilityState()).toBe("available"); + }); + + it("fails closed on malformed successful results without disabling capability", async () => { + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [["ok", "existing"]], + }, + }); + + const result = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 2, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "canonical_corrupt", + legacyFallbackAllowed: false, + }); + expect(getVersionedBindingCapabilityState()).toBe("available"); + }); + + it("CAS updates the provider and rotates generation", async () => { + const mock = createMockRedis({ + operationResponses: { + [CAS_SESSION_BINDING]: [(args) => ["ok", "updated", String(args[7]), String(args[8])]], + }, + }); + + const result = await compareAndSetSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "old-generation", + providerId: 23, + redis: mock.redis, + }); + + expect(result.status).toBe("ok"); + if (result.status !== "ok") throw new Error("Expected successful CAS"); + expect(result.snapshot.providerId).toBe(23); + expect(result.snapshot.generation).not.toBe("old-generation"); + const operation = mock.evalMock.mock.calls.at(-1); + expect(operation?.[6]).toBe("old-generation"); + expect(operation?.[8]).toBe("23"); + }); + + it("returns generation conflicts without rotating global capability", async () => { + const mock = createMockRedis({ + operationResponses: { + [CAS_SESSION_BINDING]: [["conflict", "generation_mismatch"]], + }, + }); + + const result = await compareAndSetSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "stale", + providerId: 23, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "generation_mismatch", + legacyFallbackAllowed: false, + }); + expect(getVersionedBindingCapabilityState()).toBe("available"); + }); + + it("clears a provider with a tenant-scoped cooldown in the same Lua call", async () => { + const mock = createMockRedis({ + cooldownValue: "cooldown-generation", + operationResponses: { + [CLEAR_SESSION_BINDING]: [(args) => ["ok", "cleared", String(args[8]), ""]], + }, + }); + + const result = await clearSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "bound-generation", + expectedProviderId: 23, + cooldownTtlSeconds: 120, + redis: mock.redis, + }); + + expect(result.status).toBe("ok"); + if (result.status !== "ok") throw new Error("Expected successful clear"); + expect(result.snapshot.providerId).toBeNull(); + expect(result.snapshot.generation).not.toBe("bound-generation"); + const operation = mock.evalMock.mock.calls.at(-1); + expect(operation?.slice(2, 6)).toEqual([ + buildCanonicalSessionBindingKey("sid", 4), + "session:sid:provider", + "session:sid:key", + buildSessionProviderCooldownKey("sid", 4, 23), + ]); + expect(operation?.slice(-3)).toEqual(["300", "23", "120"]); + + const cooldown = await isSessionProviderCoolingDown({ + sessionId: "sid", + keyId: 4, + providerId: 23, + redis: mock.redis, + }); + expect(cooldown).toEqual({ + status: "ok", + coolingDown: true, + legacyFallbackAllowed: false, + }); + }); + + it("rotates a null tombstone without creating a cooldown key", async () => { + const mock = createMockRedis({ + operationResponses: { + [CLEAR_SESSION_BINDING]: [(args) => ["ok", "cleared", String(args[8]), ""]], + }, + }); + + const result = await clearSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "null-generation", + expectedProviderId: null, + redis: mock.redis, + }); + + expect(result.status).toBe("ok"); + const operation = mock.evalMock.mock.calls.at(-1); + expect(operation?.[5]).toBe(buildCanonicalSessionBindingKey("sid", 4)); + expect(operation?.slice(-3)).toEqual(["300", "", "0"]); + }); + + it("rejects a cooldown without an expected provider before touching Redis", async () => { + const mock = createMockRedis(); + + const result = await clearSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "generation", + expectedProviderId: null, + cooldownTtlSeconds: 30, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "invalid_input", + legacyFallbackAllowed: false, + }); + expect(mock.evalMock).not.toHaveBeenCalled(); + }); + + it("marks operation errors unavailable and allows only infrastructure fallback", async () => { + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [new Error("ERR script execution disabled")], + }, + }); + + const first = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + const callsAfterFailure = mock.evalMock.mock.calls.length; + const second = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + + expect(first).toMatchObject({ + status: "unavailable", + reason: "capability_unavailable", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + }); + expect(second).toMatchObject({ + status: "unavailable", + reason: "capability_unavailable", + legacyFallbackAllowed: true, + }); + expect(mock.evalMock).toHaveBeenCalledTimes(callsAfterFailure); + }); + + it("fails closed on malformed binding data without disabling the capability", async () => { + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [ + ["ok", "existing", "generation-a", "invalid-provider"], + ["ok", "existing", "generation-b", "8"], + ], + }, + }); + + const first = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + const second = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + + expect(first).toEqual({ + status: "conflict", + reason: "canonical_corrupt", + legacyFallbackAllowed: false, + }); + expect(second.status).toBe("ok"); + expect(getVersionedBindingCapabilityState()).toBe("available"); + }); + + it("rejects a response from an obsolete connection epoch", async () => { + let resolveOperation: ((value: unknown) => void) | undefined; + const operation = new Promise((resolve) => { + resolveOperation = resolve; + }); + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [() => operation], + }, + }); + + const pending = readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + await vi.waitFor(() => expect(mock.evalMock).toHaveBeenCalledTimes(4)); + mock.emit("reconnecting"); + resolveOperation?.(["ok", "existing", "generation", "8"]); + + await expect(pending).resolves.toMatchObject({ + status: "unavailable", + reason: "connection_changed", + legacyFallbackAllowed: false, + }); + expect(getVersionedBindingCapabilityState()).toBe("unknown"); + }); + + it("does not run Lua while Redis is not ready", async () => { + const mock = createMockRedis({ status: "connecting" }); + + const result = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + + expect(result).toMatchObject({ + status: "unavailable", + reason: "redis_not_ready", + legacyFallbackAllowed: true, + }); + expect(mock.evalMock).not.toHaveBeenCalled(); + }); + + it("treats invalid identities as non-fallback conflicts", async () => { + const mock = createMockRedis(); + + const result = await readOrReconcileSessionBinding({ + sessionId: "", + keyId: 0, + ttlSeconds: -1, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "invalid_input", + legacyFallbackAllowed: false, + }); + expect(mock.evalMock).not.toHaveBeenCalled(); + }); + + it("returns false when a provider has no cooldown marker", async () => { + const mock = createMockRedis({ cooldownValue: null }); + + const result = await isSessionProviderCoolingDown({ + sessionId: "sid", + keyId: 4, + providerId: 9, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "ok", + coolingDown: false, + legacyFallbackAllowed: false, + }); + }); + + it("rejects foreign legacy state before any fallback mutation", async () => { + const mock = createMockRedis(); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return "foreign-key"; + if (key === "session:sid:provider") return "9"; + return null; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "set", providerId: 10 }, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "foreign_legacy_owner", + legacyFallbackAllowed: false, + }); + expect(mock.setexMock).not.toHaveBeenCalled(); + }); + + it("claims a truly empty legacy owner with NX and rechecks it", async () => { + const mock = createMockRedis(); + let owner: string | null = null; + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return owner; + return null; + }); + mock.setMock.mockImplementation(async (_key: string, value: string) => { + owner = value; + return "OK"; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "inspect" }, + }); + + expect(result).toEqual({ status: "ok", changed: false, providerId: null }); + expect(mock.setMock).toHaveBeenCalledWith("session:sid:key", "4", "EX", 300, "NX"); + }); + + it("blocks legacy mutation when canonical state already exists", async () => { + const mock = createMockRedis(); + mock.existsMock.mockResolvedValue(1); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "clear", expectedProviderId: 9 }, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "canonical_exists", + legacyFallbackAllowed: false, + }); + expect(mock.getMock).not.toHaveBeenCalled(); + }); + + it("uses the tenant-authorized termination primitive and leaves a tombstone", async () => { + const mock = createMockRedis({ + operationResponses: { + [TERMINATE_SESSION_BINDING]: [(args) => ["ok", "terminated", String(args[6]), ""]], + }, + }); + + const result = await terminateSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedProviderId: 9, + redis: mock.redis, + }); + + expect(result).toMatchObject({ + status: "ok", + source: "terminated", + snapshot: { sessionId: "sid", keyId: 4, providerId: null }, + }); + expect(mock.evalMock.mock.calls.at(-1)?.[8]).toBe("9"); + }); + + it("uses EVALSHA after capability warmup and falls back on NOSCRIPT", async () => { + const mock = createMockRedis({ + evalSha: true, + evalShaNoScriptOnce: true, + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [ + ["ok", "existing", "generation-a", "8"], + ["ok", "existing", "generation-b", "8"], + ], + }, + }); + + const first = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + const second = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + + expect(first.status).toBe("ok"); + expect(second.status).toBe("ok"); + expect(mock.evalShaMock).toHaveBeenCalled(); + expect(mock.evalMock).toHaveBeenCalled(); + }); + + it("covers tenant-safe legacy refresh, bind, set, clear, and terminate mutations", async () => { + const mock = createMockRedis(); + let owner: string | null = "4"; + let provider: string | null = null; + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return owner; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.setMock.mockImplementation(async (key: string, value: string) => { + if (key === "session:sid:key") owner = value; + if (key === "session:sid:provider") provider = value; + return "OK"; + }); + mock.setexMock.mockImplementation(async (key: string, _ttl: number, value: string) => { + if (key === "session:sid:provider") provider = value; + if (key === "session:sid:key") owner = value; + return "OK"; + }); + mock.delMock.mockImplementation(async (key: string) => { + if (key === "session:sid:provider") provider = null; + if (key === "session:sid:key") owner = null; + return 1; + }); + + await expect( + mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "refresh" }, + }) + ).resolves.toMatchObject({ status: "ok" }); + await expect( + mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "bind_if_absent", providerId: 8 }, + }) + ).resolves.toMatchObject({ status: "ok", changed: true, providerId: 8 }); + await expect( + mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "set", providerId: 9 }, + }) + ).resolves.toMatchObject({ status: "ok", changed: true, providerId: 9 }); + await expect( + mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "clear", expectedProviderId: 9 }, + }) + ).resolves.toMatchObject({ status: "ok", changed: true, providerId: null }); + + provider = "10"; + await expect( + mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "terminate", expectedProviderIds: [10] }, + }) + ).resolves.toMatchObject({ status: "ok", changed: true, providerId: null }); + expect(owner).toBeNull(); + }); + + it("rejects malformed legacy mutation arguments before touching Redis", async () => { + const mock = createMockRedis(); + + await expect( + mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "set", providerId: 0 }, + }) + ).resolves.toMatchObject({ status: "conflict", reason: "invalid_input" }); + await expect( + mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "terminate", expectedProviderIds: [0] }, + }) + ).resolves.toMatchObject({ status: "conflict", reason: "invalid_input" }); + expect(mock.existsMock).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/lib/session-manager-binding-smart.test.ts b/tests/unit/lib/session-manager-binding-smart.test.ts index 8186ff9b8..61fc72c95 100644 --- a/tests/unit/lib/session-manager-binding-smart.test.ts +++ b/tests/unit/lib/session-manager-binding-smart.test.ts @@ -21,6 +21,15 @@ let lastPipeline: { exec: ReturnType; }; +const bindingMocks = vi.hoisted(() => ({ + clearSessionBinding: vi.fn(), + compareAndSetSessionBinding: vi.fn(), + isSessionProviderCoolingDown: vi.fn(), + mutateLegacySessionBindingSafely: vi.fn(), + readOrReconcileSessionBinding: vi.fn(), + refreshSessionBinding: vi.fn(), +})); + const makePipeline = () => { const pipeline = { setex: vi.fn(() => pipeline), @@ -44,6 +53,11 @@ vi.mock("@/lib/redis", () => ({ getRedisClient: () => redisClientRef, })); +vi.mock("@/lib/redis/session-binding", () => ({ + ...bindingMocks, + getVersionedBindingCapabilityState: () => "unavailable", +})); + // Both are loaded via `await import(...)` inside updateSessionBindingSmart; the // static vi.mock still intercepts the dynamic import. vi.mock("@/repository/provider", () => ({ @@ -59,10 +73,35 @@ import { SessionManager } from "@/lib/session-manager"; import { findProviderById } from "@/repository/provider"; const SID = "sess-binding"; -const TTL = 300; +const KEY_ID = 42; +let legacyProviderId: number | null; beforeEach(() => { vi.clearAllMocks(); + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "unavailable", + reason: "capability_unavailable", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + }); + legacyProviderId = null; + bindingMocks.mutateLegacySessionBindingSafely.mockImplementation(async (input: any) => { + if (input.mutation.type === "inspect") { + return { status: "ok", changed: false, providerId: legacyProviderId }; + } + if (input.mutation.type === "bind_if_absent") { + if (legacyProviderId !== null) { + return { status: "ok", changed: false, providerId: legacyProviderId }; + } + legacyProviderId = input.mutation.providerId; + return { status: "ok", changed: true, providerId: legacyProviderId }; + } + if (input.mutation.type === "set") { + legacyProviderId = input.mutation.providerId; + return { status: "ok", changed: true, providerId: legacyProviderId }; + } + throw new Error(`Unexpected mutation ${input.mutation.type}`); + }); redisClientRef = { status: "ready", get: vi.fn(async () => null), @@ -75,7 +114,7 @@ beforeEach(() => { describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { it("forceUpdate=true overrides a healthy higher-priority existing binding", async () => { // Existing binding -> provider 1 (healthy, higher priority than the winner) - redisClientRef!.get.mockResolvedValue("1"); + legacyProviderId = 1; vi.mocked(findProviderById).mockResolvedValue({ id: 1, name: "main", priority: 5 } as never); vi.mocked(isCircuitOpen).mockResolvedValue(false); @@ -85,20 +124,20 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { 10, // winner priority (lower priority than current's 5) false, // isFirstAttempt false, // isFailoverSuccess - null, + KEY_ID, true // forceUpdate ); expect(result).toMatchObject({ updated: true, reason: "race_winner_forced" }); - expect(lastPipeline.setex).toHaveBeenCalledWith(`session:${SID}:provider`, TTL, "2"); - // Guard against a regression that queues setex but forgets to flush the pipeline. - expect(lastPipeline.exec).toHaveBeenCalledTimes(1); + expect(bindingMocks.mutateLegacySessionBindingSafely).toHaveBeenCalledWith( + expect.objectContaining({ mutation: { type: "set", providerId: 2 } }) + ); }); it("forceUpdate=true rebinds even when the winner equals the current binding", async () => { // Production winner==initialProvider race: the bound provider is already the winner, // but the race result must still (re)write the binding and refresh its TTL. - redisClientRef!.get.mockResolvedValue("2"); + legacyProviderId = 2; const result = await SessionManager.updateSessionBindingSmart( SID, @@ -106,18 +145,18 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { 10, false, false, - null, + KEY_ID, true // forceUpdate ); expect(result).toMatchObject({ updated: true, reason: "race_winner_forced" }); - expect(redisClientRef!.get).not.toHaveBeenCalled(); - expect(lastPipeline.setex).toHaveBeenCalledWith(`session:${SID}:provider`, TTL, "2"); - expect(lastPipeline.exec).toHaveBeenCalledTimes(1); + expect(bindingMocks.mutateLegacySessionBindingSafely).toHaveBeenCalledWith( + expect.objectContaining({ mutation: { type: "set", providerId: 2 } }) + ); }); it("forceUpdate=false keeps the healthy higher-priority binding (documents the gap)", async () => { - redisClientRef!.get.mockResolvedValue("1"); + legacyProviderId = 1; vi.mocked(findProviderById).mockResolvedValue({ id: 1, name: "main", priority: 5 } as never); vi.mocked(isCircuitOpen).mockResolvedValue(false); @@ -127,7 +166,7 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { 10, false, false, - null, + KEY_ID, false // forceUpdate ); @@ -135,14 +174,14 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { }); it("forceUpdate=true short-circuits before consulting provider/circuit state", async () => { - redisClientRef!.get.mockResolvedValue("1"); + legacyProviderId = 1; - await SessionManager.updateSessionBindingSmart(SID, 2, 10, false, false, null, true); + await SessionManager.updateSessionBindingSmart(SID, 2, 10, false, false, KEY_ID, true); expect(findProviderById).not.toHaveBeenCalled(); expect(isCircuitOpen).not.toHaveBeenCalled(); // forceUpdate goes straight to the unconditional pipeline path. - expect(redisClientRef!.get).not.toHaveBeenCalled(); + expect(findProviderById).not.toHaveBeenCalled(); }); it("forceUpdate=true also persists the keyId binding with TTL", async () => { @@ -152,14 +191,18 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { 10, false, false, - 42, // keyId + KEY_ID, true ); expect(result.updated).toBe(true); - expect(lastPipeline.setex).toHaveBeenCalledWith(`session:${SID}:provider`, TTL, "2"); - expect(lastPipeline.setex).toHaveBeenCalledWith(`session:${SID}:key`, TTL, "42"); - expect(lastPipeline.exec).toHaveBeenCalledTimes(1); + expect(bindingMocks.mutateLegacySessionBindingSafely).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SID, + keyId: KEY_ID, + mutation: { type: "set", providerId: 2 }, + }) + ); }); it("isFailoverSuccess=true keeps reason failover_success even when forceUpdate=true", async () => { @@ -169,7 +212,7 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { 10, false, true, // isFailoverSuccess - null, + KEY_ID, true // forceUpdate ); @@ -191,4 +234,77 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { expect(result).toMatchObject({ updated: false, reason: "redis_not_ready" }); }); + + it("uses generation CAS instead of legacy writes when versioned binding is available", async () => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "ok", + source: "existing", + snapshot: { + sessionId: SID, + keyId: 42, + providerId: 1, + generation: "generation-a", + }, + legacyFallbackAllowed: false, + }); + bindingMocks.compareAndSetSessionBinding.mockResolvedValue({ + status: "ok", + source: "updated", + snapshot: { + sessionId: SID, + keyId: 42, + providerId: 2, + generation: "generation-b", + }, + legacyFallbackAllowed: false, + }); + + const result = await SessionManager.updateSessionBindingSmart( + SID, + 2, + 10, + false, + false, + 42, + true + ); + + expect(result).toMatchObject({ updated: true, reason: "race_winner_forced" }); + expect(bindingMocks.compareAndSetSessionBinding).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SID, + keyId: 42, + expectedGeneration: "generation-a", + providerId: 2, + }) + ); + expect(redisClientRef!.pipeline).not.toHaveBeenCalled(); + expect(redisClientRef!.setex).not.toHaveBeenCalled(); + }); + + it("does not fall back to legacy writes for a foreign owner conflict", async () => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "conflict", + reason: "foreign_legacy_owner", + legacyFallbackAllowed: false, + }); + + const result = await SessionManager.updateSessionBindingSmart( + SID, + 2, + 10, + false, + false, + 42, + true + ); + + expect(result).toEqual({ + updated: false, + reason: "versioned_binding_conflict", + details: "foreign_legacy_owner", + }); + expect(redisClientRef!.pipeline).not.toHaveBeenCalled(); + expect(redisClientRef!.set).not.toHaveBeenCalled(); + }); }); diff --git a/tests/unit/lib/session-manager-terminate-provider-sessions.test.ts b/tests/unit/lib/session-manager-terminate-provider-sessions.test.ts index 665a979c0..30d346484 100644 --- a/tests/unit/lib/session-manager-terminate-provider-sessions.test.ts +++ b/tests/unit/lib/session-manager-terminate-provider-sessions.test.ts @@ -50,6 +50,9 @@ describe("SessionManager.terminateProviderSessionsBatch", () => { expect(pipelineRef.zrange).toHaveBeenCalledTimes(2); expect(pipelineRef.zrange).toHaveBeenCalledWith("provider:42:active_sessions", 0, -1); expect(pipelineRef.zrange).toHaveBeenCalledWith("provider:43:active_sessions", 0, -1); - expect(terminateSessionsBatchSpy).toHaveBeenCalledWith(["sess-a", "sess-b", "sess-c"]); + expect(terminateSessionsBatchSpy).toHaveBeenCalledWith( + ["sess-a", "sess-b", "sess-c"], + [42, 43] + ); }); }); diff --git a/tests/unit/lib/session-manager-terminate-session.test.ts b/tests/unit/lib/session-manager-terminate-session.test.ts index 76145bd34..ffb32b9c4 100644 --- a/tests/unit/lib/session-manager-terminate-session.test.ts +++ b/tests/unit/lib/session-manager-terminate-session.test.ts @@ -2,6 +2,15 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; let redisClientRef: any; let pipelineRef: any; +const bindingMocks = vi.hoisted(() => ({ + clearSessionBinding: vi.fn(), + compareAndSetSessionBinding: vi.fn(), + isSessionProviderCoolingDown: vi.fn(), + mutateLegacySessionBindingSafely: vi.fn(), + readOrReconcileSessionBinding: vi.fn(), + refreshSessionBinding: vi.fn(), + terminateSessionBinding: vi.fn(), +})); vi.mock("server-only", () => ({})); @@ -19,6 +28,11 @@ vi.mock("@/lib/redis", () => ({ getRedisClient: () => redisClientRef, })); +vi.mock("@/lib/redis/session-binding", () => ({ + ...bindingMocks, + getVersionedBindingCapabilityState: () => "unavailable", +})); + describe("SessionManager.terminateSession", () => { beforeEach(() => { vi.resetAllMocks(); @@ -39,6 +53,17 @@ describe("SessionManager.terminateSession", () => { eval: vi.fn(async () => 1), pipeline: vi.fn(() => pipelineRef), }; + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "unavailable", + reason: "capability_unavailable", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + }); + bindingMocks.mutateLegacySessionBindingSafely.mockResolvedValue({ + status: "ok", + changed: true, + providerId: null, + }); }); it("应同时从 global/key/user 的 active_sessions ZSET 中移除 sessionId（若可解析到 userId）", async () => { @@ -89,22 +114,69 @@ describe("SessionManager.terminateSession", () => { it("迟到 cleanup 仅删除仍绑定到预期 provider 的 session", async () => { const { SessionManager } = await import("@/lib/session-manager"); - await expect(SessionManager.clearSessionProvider("sess_compare", 42)).resolves.toBe(true); + await expect(SessionManager.clearSessionProvider("sess_compare", 42, 7)).resolves.toBe(true); - expect(redisClientRef.eval).toHaveBeenCalledWith( - expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'), - 1, - "session:sess_compare:provider", - "42" + expect(bindingMocks.mutateLegacySessionBindingSafely).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: "sess_compare", + keyId: 7, + mutation: { type: "clear", expectedProviderId: 42 }, + }) ); - expect(redisClientRef.del).not.toHaveBeenCalled(); }); it("迟到 cleanup 不删除已切换到新 provider 的 session", async () => { - redisClientRef.eval.mockResolvedValueOnce(0); + bindingMocks.mutateLegacySessionBindingSafely.mockResolvedValueOnce({ + status: "conflict", + reason: "provider_mismatch", + legacyFallbackAllowed: false, + }); const { SessionManager } = await import("@/lib/session-manager"); - await expect(SessionManager.clearSessionProvider("sess_compare", 42)).resolves.toBe(false); + await expect(SessionManager.clearSessionProvider("sess_compare", 42, 7)).resolves.toBe(false); expect(redisClientRef.del).not.toHaveBeenCalled(); }); + + it("versioned termination leaves an owned tombstone instead of deleting mirrors", async () => { + const sessionId = "sess_versioned"; + redisClientRef.get.mockImplementation(async (key: string) => { + if (key === `session:${sessionId}:provider`) return "42"; + if (key === `session:${sessionId}:key`) return "7"; + return null; + }); + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "ok", + source: "existing", + snapshot: { + sessionId, + keyId: 7, + providerId: 42, + generation: "generation-a", + }, + legacyFallbackAllowed: false, + }); + bindingMocks.terminateSessionBinding.mockResolvedValue({ + status: "ok", + source: "terminated", + snapshot: { + sessionId, + keyId: 7, + providerId: null, + generation: "generation-b", + }, + legacyFallbackAllowed: false, + }); + const { SessionManager } = await import("@/lib/session-manager"); + + await expect(SessionManager.terminateSession(sessionId)).resolves.toBe(true); + + expect(bindingMocks.terminateSessionBinding).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId, + keyId: 7, + }) + ); + expect(pipelineRef.del).not.toHaveBeenCalledWith(`session:${sessionId}:provider`); + expect(pipelineRef.del).not.toHaveBeenCalledWith(`session:${sessionId}:key`); + }); }); diff --git a/tests/unit/lib/session-manager-versioned-binding.test.ts b/tests/unit/lib/session-manager-versioned-binding.test.ts new file mode 100644 index 000000000..57d9f5252 --- /dev/null +++ b/tests/unit/lib/session-manager-versioned-binding.test.ts @@ -0,0 +1,151 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const bindingMocks = vi.hoisted(() => ({ + clearSessionBinding: vi.fn(), + compareAndSetSessionBinding: vi.fn(), + isSessionProviderCoolingDown: vi.fn(), + readOrReconcileSessionBinding: vi.fn(), + refreshSessionBinding: vi.fn(), +})); + +let redisClientRef: { + status: string; + del: ReturnType; + eval: ReturnType; + get: ReturnType; + pipeline: ReturnType; + set: ReturnType; + setex: ReturnType; +}; + +vi.mock("server-only", () => ({})); +vi.mock("@/lib/logger", () => ({ + logger: { + debug: vi.fn(), + error: vi.fn(), + info: vi.fn(), + trace: vi.fn(), + warn: vi.fn(), + }, +})); +vi.mock("@/lib/redis", () => ({ + getRedisClient: () => redisClientRef, +})); +vi.mock("@/lib/redis/session-binding", () => ({ + ...bindingMocks, + getVersionedBindingCapabilityState: () => "available", +})); + +import { SessionManager } from "@/lib/session-manager"; + +const SESSION_ID = "session-versioned"; +const KEY_ID = 7; +const PROVIDER_ID = 42; + +function snapshot(providerId: number | null = PROVIDER_ID) { + return { + status: "ok" as const, + source: "existing" as const, + snapshot: { + sessionId: SESSION_ID, + keyId: KEY_ID, + providerId, + generation: "generation-a", + }, + legacyFallbackAllowed: false as const, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + const pipeline = { + expire: vi.fn(), + exec: vi.fn(async () => []), + setex: vi.fn(), + }; + pipeline.expire.mockReturnValue(pipeline); + pipeline.setex.mockReturnValue(pipeline); + redisClientRef = { + status: "ready", + del: vi.fn(async () => 1), + eval: vi.fn(async () => 1), + get: vi.fn(async () => null), + pipeline: vi.fn(() => pipeline), + set: vi.fn(async () => "OK"), + setex: vi.fn(async () => "OK"), + }; + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue(snapshot()); +}); + +describe("SessionManager versioned binding adapter", () => { + it("returns the tenant-scoped provider without reading the legacy mirror", async () => { + await expect(SessionManager.getSessionProvider(SESSION_ID, KEY_ID)).resolves.toBe(PROVIDER_ID); + + expect(bindingMocks.readOrReconcileSessionBinding).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: SESSION_ID, keyId: KEY_ID }) + ); + expect(redisClientRef.get).not.toHaveBeenCalled(); + }); + + it("fails closed on a foreign legacy owner", async () => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "conflict", + reason: "foreign_legacy_owner", + legacyFallbackAllowed: false, + }); + + await expect(SessionManager.getSessionProvider(SESSION_ID, KEY_ID)).resolves.toBeNull(); + + expect(redisClientRef.get).not.toHaveBeenCalled(); + }); + + it("clears through generation CAS and does not delete the legacy provider directly", async () => { + bindingMocks.clearSessionBinding.mockResolvedValue({ + status: "ok", + source: "cleared", + snapshot: { + sessionId: SESSION_ID, + keyId: KEY_ID, + providerId: null, + generation: "generation-b", + }, + legacyFallbackAllowed: false, + }); + + await expect( + SessionManager.clearSessionProvider(SESSION_ID, PROVIDER_ID, KEY_ID) + ).resolves.toBe(true); + + expect(bindingMocks.clearSessionBinding).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SESSION_ID, + keyId: KEY_ID, + expectedGeneration: "generation-a", + expectedProviderId: PROVIDER_ID, + }) + ); + expect(redisClientRef.del).not.toHaveBeenCalled(); + expect(redisClientRef.eval).not.toHaveBeenCalled(); + }); + + it("does not let a Codex cache key claim a foreign legacy session", async () => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "conflict", + reason: "foreign_legacy_owner", + legacyFallbackAllowed: false, + }); + + await expect( + SessionManager.updateSessionWithCodexCacheKey( + "current-session", + "shared-cache-key", + PROVIDER_ID, + KEY_ID + ) + ).resolves.toEqual({ sessionId: "current-session", updated: false }); + + expect(redisClientRef.get).not.toHaveBeenCalled(); + expect(redisClientRef.pipeline).not.toHaveBeenCalled(); + expect(redisClientRef.setex).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/lib/session-tracker-cleanup.test.ts b/tests/unit/lib/session-tracker-cleanup.test.ts index e13f718ab..02a1cb046 100644 --- a/tests/unit/lib/session-tracker-cleanup.test.ts +++ b/tests/unit/lib/session-tracker-cleanup.test.ts @@ -74,7 +74,6 @@ describe("SessionTracker - TTL and cleanup", () => { pipelineCalls.length = 0; vi.useFakeTimers(); vi.setSystemTime(new Date(nowMs)); - redisClientRef = { status: "ready", exists: vi.fn(async () => 1), @@ -163,30 +162,17 @@ describe("SessionTracker - TTL and cleanup", () => { expect(providerExpireCall![2]).toBe(7200); // should use SESSION_TTL when > 3600 }); - it("should refresh session binding TTLs using env SESSION_TTL (not hardcoded 300)", async () => { + it("should refresh session activity using env SESSION_TTL (not hardcoded 300)", async () => { process.env.SESSION_TTL = "600"; // 10 minutes const { SessionTracker } = await import("@/lib/session-tracker"); await SessionTracker.refreshSession("sess-123", 1, 42); - // Check expire calls for session bindings use 600 (env value), not 300 - const providerBindingExpire = pipelineCalls.find( - (call) => call[0] === "expire" && String(call[1]) === "session:sess-123:provider" - ); - const keyBindingExpire = pipelineCalls.find( - (call) => call[0] === "expire" && String(call[1]) === "session:sess-123:key" - ); const lastSeenSetex = pipelineCalls.find( (call) => call[0] === "setex" && String(call[1]) === "session:sess-123:last_seen" ); - expect(providerBindingExpire).toBeDefined(); - expect(providerBindingExpire![2]).toBe(600); - - expect(keyBindingExpire).toBeDefined(); - expect(keyBindingExpire![2]).toBe(600); - expect(lastSeenSetex).toBeDefined(); expect(lastSeenSetex![2]).toBe(600); }); diff --git a/tests/unit/proxy/provider-selector-cross-type-model.test.ts b/tests/unit/proxy/provider-selector-cross-type-model.test.ts index f171f6ffe..94eaec709 100644 --- a/tests/unit/proxy/provider-selector-cross-type-model.test.ts +++ b/tests/unit/proxy/provider-selector-cross-type-model.test.ts @@ -303,7 +303,8 @@ describe("findReusable - cross-type model routing (#832)", () => { expect(result).toBeNull(); expect(sessionManagerMocks.SessionManager.clearSessionProvider).toHaveBeenCalledWith( "cross-type-3", - 12 + 12, + null ); }); @@ -336,7 +337,8 @@ describe("findReusable - cross-type model routing (#832)", () => { expect(result).toBeNull(); expect(sessionManagerMocks.SessionManager.clearSessionProvider).toHaveBeenCalledWith( "cross-type-6", - 15 + 15, + null ); }); }); diff --git a/tests/unit/proxy/provider-selector-model-mismatch-binding.test.ts b/tests/unit/proxy/provider-selector-model-mismatch-binding.test.ts index d21486433..5efbe753a 100644 --- a/tests/unit/proxy/provider-selector-model-mismatch-binding.test.ts +++ b/tests/unit/proxy/provider-selector-model-mismatch-binding.test.ts @@ -116,7 +116,8 @@ describe("findReusable - model mismatch clears stale binding", () => { expect(result).toBeNull(); expect(sessionManagerMocks.SessionManager.clearSessionProvider).toHaveBeenCalledWith( "sess_disable_reuse", - 78 + 78, + null ); }); @@ -141,7 +142,8 @@ describe("findReusable - model mismatch clears stale binding", () => { // Key assertion: clearSessionProvider should have been called expect(sessionManagerMocks.SessionManager.clearSessionProvider).toHaveBeenCalledWith( "4c25cf92", - 78 + 78, + null ); }); @@ -165,7 +167,8 @@ describe("findReusable - model mismatch clears stale binding", () => { expect(result).toBeNull(); expect(sessionManagerMocks.SessionManager.clearSessionProvider).toHaveBeenCalledWith( "sess_response_format_mismatch", - 94 + 94, + null ); }); @@ -240,7 +243,8 @@ describe("findReusable - model mismatch clears stale binding", () => { expect(result).toBeNull(); expect(sessionManagerMocks.SessionManager.clearSessionProvider).toHaveBeenCalledWith( "sess_variant", - 78 + 78, + null ); }); }); diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index 014219a24..fe5d7f4a8 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -13,6 +13,7 @@ const mocks = vi.hoisted(() => ({ updateSessionBindingSmart: vi.fn(async () => ({ updated: true, reason: "test" })), updateSessionProvider: vi.fn(async () => {}), clearSessionProvider: vi.fn(async () => {}), + clearSessionProviders: vi.fn(async () => false), isHttp2Enabled: vi.fn(async () => false), getPreferredProviderEndpoints: vi.fn(async () => []), getEndpointFilterStats: vi.fn(async () => null), @@ -92,6 +93,7 @@ vi.mock("@/lib/session-manager", () => ({ updateSessionBindingSmart: mocks.updateSessionBindingSmart, updateSessionProvider: mocks.updateSessionProvider, clearSessionProvider: mocks.clearSessionProvider, + clearSessionProviders: mocks.clearSessionProviders, storeSessionSpecialSettings: mocks.storeSessionSpecialSettings, storeSessionRequestPhaseSnapshot: mocks.storeSessionRequestPhaseSnapshot, storeSessionResponsePhaseSnapshot: mocks.storeSessionResponsePhaseSnapshot, @@ -1454,7 +1456,7 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { await rejection; expect(controller1.signal.aborted).toBe(true); expect(controller2.signal.aborted).toBe(true); - expect(mocks.clearSessionProvider).toHaveBeenCalledWith("sess-hedge", 1); + expect(mocks.clearSessionProviders).toHaveBeenCalledWith("sess-hedge", new Set([1, 2]), null); expect(mocks.recordFailure).not.toHaveBeenCalled(); expect(mocks.recordSuccess).not.toHaveBeenCalled(); @@ -1646,7 +1648,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { expect(error.message).toBe("所有供应商暂时不可用，请稍后重试"); expect(error.message).not.toContain("invalid key"); expect(error.message).not.toContain("model not found"); - expect(mocks.clearSessionProvider).toHaveBeenCalledWith("sess-hedge", 1); + expect(mocks.clearSessionProviders).toHaveBeenCalledWith( + "sess-hedge", + new Set([1, 2]), + null + ); } finally { vi.useRealTimers(); } @@ -1694,7 +1700,7 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { expect(error.message).toBe("prompt too long"); expect(doForward).toHaveBeenCalledTimes(1); expect(mocks.pickRandomProviderWithExclusion).not.toHaveBeenCalled(); - expect(mocks.clearSessionProvider).toHaveBeenCalledWith("sess-hedge", 1); + expect(mocks.clearSessionProviders).toHaveBeenCalledWith("sess-hedge", new Set([1]), null); expect(session.getProviderChain()).toEqual( expect.arrayContaining([ expect.objectContaining({ @@ -1751,7 +1757,7 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { expect(mocks.pickRandomProviderWithExclusion).not.toHaveBeenCalled(); expect(mocks.recordEndpointFailure).not.toHaveBeenCalled(); expect(mocks.recordFailure).not.toHaveBeenCalled(); - expect(mocks.clearSessionProvider).toHaveBeenCalledWith("sess-hedge", 1); + expect(mocks.clearSessionProviders).toHaveBeenCalledWith("sess-hedge", new Set([1]), null); expect(session.getProviderChain()).toEqual([ expect.objectContaining({ id: provider.id, diff --git a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts index c3bdf3a5d..af432f126 100644 --- a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts +++ b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts @@ -357,7 +357,12 @@ function setupCommonMocks() { updatedAt: new Date(), }); vi.mocked(updateMessageRequestDetails).mockResolvedValue(undefined); - vi.mocked(updateMessageRequestDetailsDurably).mockResolvedValue(true); + vi.mocked(updateMessageRequestDetailsDurably).mockImplementation( + async (_messageId, _details, options) => { + await options?.onCommitted?.(); + return true; + } + ); vi.mocked(updateMessageRequestDuration).mockResolvedValue(undefined); vi.mocked(SessionManager.storeSessionResponse).mockResolvedValue(undefined); vi.mocked(SessionManager.clearSessionProvider).mockResolvedValue(undefined); @@ -409,7 +414,7 @@ describe("Endpoint circuit breaker isolation", () => { expect.objectContaining({ message: expect.stringContaining("FAKE_200") }) ); expect(mockRecordEndpointFailure).not.toHaveBeenCalled(); - expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1); + expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1, 456); const chain = session.getProviderChain(); expect( @@ -438,7 +443,7 @@ describe("Endpoint circuit breaker isolation", () => { expect.objectContaining({ message: expect.stringContaining("FAKE_200") }) ); expect(mockRecordEndpointFailure).not.toHaveBeenCalled(); - expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1); + expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1, 456); expect(SessionManager.updateSessionUsage).not.toHaveBeenCalled(); expect(SessionTracker.refreshSession).not.toHaveBeenCalled(); }); @@ -454,7 +459,7 @@ describe("Endpoint circuit breaker isolation", () => { expect(mockRecordFailure).not.toHaveBeenCalled(); expect(mockRecordEndpointFailure).not.toHaveBeenCalled(); - expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1); + expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1, 456); const chain = session.getProviderChain(); expect( From 0e90937fd67374967cca2ea96365d93578360415 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 13:45:25 -0400 Subject: [PATCH 02/89] fix: observe late capability probe failures --- src/lib/redis/session-binding.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/redis/session-binding.ts b/src/lib/redis/session-binding.ts index eb6037a5f..54a58de5c 100644 --- a/src/lib/redis/session-binding.ts +++ b/src/lib/redis/session-binding.ts @@ -576,6 +576,10 @@ async function withCapabilityProbeDeadline( operation: Promise, deadlineAt: number ): Promise { + // The timeout races the Redis command but cannot cancel it. Observe a late + // rejection so an operation that settles after the deadline never becomes + // an unhandled promise rejection. + operation.catch(() => {}); const remainingMs = deadlineAt - Date.now(); if (remainingMs <= 0) throw new Error("Session binding capability probe deadline exceeded"); From fa8342c408c0f893c83b114447e3b5eb91048e66 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 13:55:37 -0400 Subject: [PATCH 03/89] fix: guard versioned session termination conflicts --- src/lib/session-manager.ts | 12 ++++++++ tests/unit/lib/redis/session-binding.test.ts | 24 +++++++++++++++ .../session-manager-terminate-session.test.ts | 30 +++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/src/lib/session-manager.ts b/src/lib/session-manager.ts index b47b9a3bf..3cadabd6e 100644 --- a/src/lib/session-manager.ts +++ b/src/lib/session-manager.ts @@ -2843,6 +2843,18 @@ export class SessionManager { return false; } + // A binding-aware termination must succeed before any session metadata or + // active-session indexes are removed. This guard keeps a future mutation + // path from turning a CAS/mirror conflict into a misleading success based + // only on unrelated metadata deletions. + if (keyId !== null && !bindingTerminated) { + logger.warn("SessionManager: Session binding was not terminated", { + sessionId, + keyId, + }); + return false; + } + // 2. 删除所有 Session 相关的 key const pipeline = redis.pipeline(); diff --git a/tests/unit/lib/redis/session-binding.test.ts b/tests/unit/lib/redis/session-binding.test.ts index 739ac07b3..09ec9072c 100644 --- a/tests/unit/lib/redis/session-binding.test.ts +++ b/tests/unit/lib/redis/session-binding.test.ts @@ -587,6 +587,30 @@ describe("versioned session binding operations", () => { expect(mock.evalMock).toHaveBeenCalledTimes(callsAfterFailure); }); + it("allows legacy fallback on the first runtime capability error", async () => { + const mock = createMockRedis({ + operationResponses: { + [CAS_SESSION_BINDING]: [new Error("ERR script execution disabled")], + }, + }); + + const result = await compareAndSetSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "generation-a", + providerId: 8, + redis: mock.redis, + }); + + expect(result).toMatchObject({ + status: "unavailable", + reason: "capability_unavailable", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + }); + expect(getVersionedBindingCapabilityState()).toBe("unavailable"); + }); + it("fails closed on malformed binding data without disabling the capability", async () => { const mock = createMockRedis({ operationResponses: { diff --git a/tests/unit/lib/session-manager-terminate-session.test.ts b/tests/unit/lib/session-manager-terminate-session.test.ts index ffb32b9c4..7467b1f5d 100644 --- a/tests/unit/lib/session-manager-terminate-session.test.ts +++ b/tests/unit/lib/session-manager-terminate-session.test.ts @@ -179,4 +179,34 @@ describe("SessionManager.terminateSession", () => { expect(pipelineRef.del).not.toHaveBeenCalledWith(`session:${sessionId}:provider`); expect(pipelineRef.del).not.toHaveBeenCalledWith(`session:${sessionId}:key`); }); + + it("does not delete session metadata when versioned termination hits a mirror conflict", async () => { + const sessionId = "sess_mirror_conflict"; + redisClientRef.get.mockImplementation(async (key: string) => { + if (key === `session:${sessionId}:provider`) return "42"; + if (key === `session:${sessionId}:key`) return "7"; + return null; + }); + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "ok", + source: "existing", + snapshot: { + sessionId, + keyId: 7, + providerId: 42, + generation: "generation-a", + }, + legacyFallbackAllowed: false, + }); + bindingMocks.terminateSessionBinding.mockResolvedValue({ + status: "conflict", + reason: "mirror_conflict", + legacyFallbackAllowed: false, + }); + const { SessionManager } = await import("@/lib/session-manager"); + + await expect(SessionManager.terminateSession(sessionId)).resolves.toBe(false); + expect(pipelineRef.exec).not.toHaveBeenCalled(); + expect(pipelineRef.del).not.toHaveBeenCalled(); + }); }); From a5aeccff5194190b9618e5175b741be1f116e653 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 14:05:22 -0400 Subject: [PATCH 04/89] feat(proxy): add bounded streaming discovery --- .../v1/_lib/proxy/discovery-coordinator.ts | 255 +++++++++ src/app/v1/_lib/proxy/discovery-validity.ts | 156 ++++++ src/app/v1/_lib/proxy/forwarder.ts | 530 +++++++++++++++++- src/app/v1/_lib/proxy/provider-selector.ts | 54 +- src/app/v1/_lib/proxy/response-handler.ts | 83 ++- src/app/v1/_lib/proxy/session.ts | 14 + src/app/v1/_lib/proxy/stream-finalization.ts | 6 + src/lib/config/system-settings-cache.ts | 7 + src/types/system-config.ts | 17 + .../unit/proxy/discovery-coordinator.test.ts | 63 +++ tests/unit/proxy/discovery-validity.test.ts | 63 +++ 11 files changed, 1235 insertions(+), 13 deletions(-) create mode 100644 src/app/v1/_lib/proxy/discovery-coordinator.ts create mode 100644 src/app/v1/_lib/proxy/discovery-validity.ts create mode 100644 tests/unit/proxy/discovery-coordinator.test.ts create mode 100644 tests/unit/proxy/discovery-validity.test.ts diff --git a/src/app/v1/_lib/proxy/discovery-coordinator.ts b/src/app/v1/_lib/proxy/discovery-coordinator.ts new file mode 100644 index 000000000..44d9de404 --- /dev/null +++ b/src/app/v1/_lib/proxy/discovery-coordinator.ts @@ -0,0 +1,255 @@ +/** + * Pure state machine for bounded provider discovery. + * + * The coordinator deliberately has no network or timer dependencies. The + * forwarder owns attempts and calls these methods at event boundaries. This + * keeps cancellation and stale-event handling deterministic and testable. + */ + +export type DiscoveryAttemptKind = "normal" | "fallback"; +export type DiscoveryState = + | "STICKY_PROBING" + | "DISCOVERY_RACING" + | "FALLBACK_READY_HELD" + | "FALLBACK_ACTIVE" + | "WINNER_COMMITTED" + | "TERMINAL_FAILED"; + +export type DiscoveryAttempt = { + id: string; + providerId: number; + priority: number; + kind: DiscoveryAttemptKind; + ready: boolean; + pending: boolean; + round: number; + launchOrder: number; +}; + +export type DiscoveryAction = + | { type: "commit_normal"; attemptId: string } + | { type: "promote_fallback"; attemptId: string } + | { type: "cancel"; attemptIds: string[] } + | { type: "launch"; slots: number } + | { type: "none" } + | { type: "terminal_failure" }; + +export type DiscoveryCoordinatorOptions = { + concurrency: number; + maxRounds: number; +}; + +function compareAttempts(a: DiscoveryAttempt, b: DiscoveryAttempt): number { + return a.priority - b.priority || a.launchOrder - b.launchOrder; +} + +export class DiscoveryCoordinator { + readonly concurrency: number; + readonly maxRounds: number; + state: DiscoveryState = "DISCOVERY_RACING"; + round = 1; + private attempts = new Map(); + private requestEpoch = 0; + private roundEpoch = 0; + + constructor(options: DiscoveryCoordinatorOptions) { + this.concurrency = Math.max(1, Math.floor(options.concurrency)); + this.maxRounds = Math.max(1, Math.floor(options.maxRounds)); + } + + get epochs(): { requestEpoch: number; roundEpoch: number } { + return { requestEpoch: this.requestEpoch, roundEpoch: this.roundEpoch }; + } + + beginRound(): { requestEpoch: number; roundEpoch: number; round: number } { + this.roundEpoch += 1; + return { ...this.epochs, round: this.round }; + } + + addAttempt(attempt: DiscoveryAttempt): boolean { + if (this.isTerminal || this.attempts.has(attempt.id)) return false; + this.attempts.set(attempt.id, { ...attempt, round: this.round }); + return true; + } + + removeAttempt(id: string): void { + this.attempts.delete(id); + } + + get isTerminal(): boolean { + return this.state === "WINNER_COMMITTED" || this.state === "TERMINAL_FAILED"; + } + + get activeAttempts(): DiscoveryAttempt[] { + return Array.from(this.attempts.values()).filter((attempt) => attempt.pending); + } + + get snapshot(): DiscoveryAttempt[] { + return Array.from(this.attempts.values()).map((attempt) => ({ ...attempt })); + } + + /** Ignore events from a cancelled request or an old round. */ + acceptsEpoch(requestEpoch: number, roundEpoch: number): boolean { + return requestEpoch === this.requestEpoch && roundEpoch === this.roundEpoch; + } + + markReady( + id: string, + requestEpoch = this.requestEpoch, + roundEpoch = this.roundEpoch + ): DiscoveryAction { + if (!this.acceptsEpoch(requestEpoch, roundEpoch) || this.isTerminal) return { type: "none" }; + const attempt = this.attempts.get(id); + if (!attempt?.pending) return { type: "none" }; + attempt.ready = true; + return this.chooseReadyNormal(); + } + + markFailed( + id: string, + requestEpoch = this.requestEpoch, + roundEpoch = this.roundEpoch + ): DiscoveryAction { + if (!this.acceptsEpoch(requestEpoch, roundEpoch) || this.isTerminal) return { type: "none" }; + const attempt = this.attempts.get(id); + if (!attempt) return { type: "none" }; + attempt.pending = false; + attempt.ready = false; + return this.afterAttemptState(); + } + + /** A normal ready result may win only after priority gating is satisfied. */ + private chooseReadyNormal(ignorePriorityGate = false): DiscoveryAction { + const readyNormal = Array.from(this.attempts.values()) + .filter((attempt) => attempt.pending && attempt.ready && attempt.kind === "normal") + .sort(compareAttempts); + if (readyNormal.length === 0) return { type: "none" }; + const bestPriority = readyNormal[0].priority; + if (!ignorePriorityGate) { + const higherTierPending = Array.from(this.attempts.values()).some( + (attempt) => + attempt.pending && + attempt.kind === "normal" && + !attempt.ready && + attempt.priority < bestPriority + ); + if (higherTierPending) return { type: "none" }; + } + const sameTier = readyNormal.filter((attempt) => attempt.priority === bestPriority); + const winner = sameTier[0]; + this.state = "WINNER_COMMITTED"; + winner.pending = false; + return { + type: "commit_normal", + attemptId: winner.id, + }; + } + + /** + * Close the current SLA window. At a boundary a ready normal always wins; + * otherwise the best still-pending normal becomes the sole fallback. A + * fallback that is merely ready is held until no normal can still win. + */ + onRoundBoundary(requestEpoch = this.requestEpoch, roundEpoch = this.roundEpoch): DiscoveryAction { + if (!this.acceptsEpoch(requestEpoch, roundEpoch) || this.isTerminal) return { type: "none" }; + const readyAction = this.chooseReadyNormal(true); + if (readyAction.type === "commit_normal") return readyAction; + + const currentFallback = Array.from(this.attempts.values()).find( + (attempt) => attempt.pending && attempt.kind === "fallback" + ); + if (currentFallback?.ready) { + currentFallback.pending = false; + this.state = "FALLBACK_ACTIVE"; + return { type: "promote_fallback", attemptId: currentFallback.id }; + } + + const pendingNormal = Array.from(this.attempts.values()) + .filter((attempt) => attempt.pending && attempt.kind === "normal") + .sort(compareAttempts); + if (currentFallback && pendingNormal.length > 0) { + for (const attempt of pendingNormal) attempt.pending = false; + if (this.round < this.maxRounds) { + this.round += 1; + this.roundEpoch += 1; + this.state = "DISCOVERY_RACING"; + return { type: "launch", slots: Math.max(1, this.concurrency - 1) }; + } + return { type: "none" }; + } + if (pendingNormal.length === 0) { + if (currentFallback) { + this.state = "FALLBACK_READY_HELD"; + return { type: "none" }; + } + return this.finishOrLaunch(); + } + + const fallback = pendingNormal[0]; + fallback.kind = "fallback"; + this.state = "FALLBACK_READY_HELD"; + const losers = pendingNormal.slice(1).map((attempt) => attempt.id); + for (const id of losers) this.attempts.get(id)!.pending = false; + + if (currentFallback) { + currentFallback.pending = true; + return { type: "cancel", attemptIds: losers }; + } + return { type: "cancel", attemptIds: losers }; + } + + onDeadline(): DiscoveryAction { + if (this.isTerminal) return { type: "none" }; + const fallback = Array.from(this.attempts.values()).find( + (attempt) => attempt.pending && attempt.kind === "fallback" && attempt.ready + ); + if (fallback) { + fallback.pending = false; + this.state = "FALLBACK_ACTIVE"; + return { type: "promote_fallback", attemptId: fallback.id }; + } + this.state = "TERMINAL_FAILED"; + return { type: "terminal_failure" }; + } + + commitWinner(id: string): DiscoveryAction { + const attempt = this.attempts.get(id); + if (!attempt || this.isTerminal) return { type: "none" }; + attempt.pending = false; + this.state = "WINNER_COMMITTED"; + return { + type: attempt.kind === "fallback" ? "promote_fallback" : "commit_normal", + attemptId: id, + }; + } + + cancelRequest(): DiscoveryAction { + this.requestEpoch += 1; + this.roundEpoch += 1; + const ids = this.activeAttempts.map((attempt) => attempt.id); + for (const attempt of this.attempts.values()) attempt.pending = false; + this.state = "TERMINAL_FAILED"; + return { type: "cancel", attemptIds: ids }; + } + + private afterAttemptState(): DiscoveryAction { + const pending = this.activeAttempts; + if (pending.length === 0) return this.finishOrLaunch(); + const fallback = pending.find((attempt) => attempt.kind === "fallback"); + if (fallback?.ready && pending.every((attempt) => attempt.kind === "fallback")) { + return this.commitWinner(fallback.id); + } + return { type: "none" }; + } + + private finishOrLaunch(): DiscoveryAction { + if (this.round >= this.maxRounds) { + this.state = "TERMINAL_FAILED"; + return { type: "terminal_failure" }; + } + this.round += 1; + this.roundEpoch += 1; + this.state = "DISCOVERY_RACING"; + return { type: "launch", slots: Math.max(1, this.concurrency - 1) }; + } +} diff --git a/src/app/v1/_lib/proxy/discovery-validity.ts b/src/app/v1/_lib/proxy/discovery-validity.ts new file mode 100644 index 000000000..29ad6a960 --- /dev/null +++ b/src/app/v1/_lib/proxy/discovery-validity.ts @@ -0,0 +1,156 @@ +export type DiscoveryProtocol = + | "anthropic" + | "openai-chat" + | "openai-responses" + | "gemini" + | "unknown"; + +export type DiscoveryValidity = { + ready: boolean; + terminal: boolean; + error: boolean; +}; + +function hasContent(value: unknown): boolean { + if (typeof value === "string") return value.trim().length > 0; + if (!value || typeof value !== "object") return false; + if (Array.isArray(value)) return value.some(hasContent); + const object = value as Record; + return [ + "text", + "content", + "delta", + "output_text", + "thinking", + "tool_use", + "tool_calls", + "functionCall", + "function_call", + "arguments", + "input", + "parts", + ].some((key) => hasContent(object[key])); +} + +function classifyJson(value: unknown, protocol: DiscoveryProtocol): DiscoveryValidity { + if (!value || typeof value !== "object") return { ready: false, terminal: false, error: true }; + const object = value as Record; + if ( + object.error || + object.failed || + object.type === "error" || + object.type === "response.failed" + ) { + return { ready: false, terminal: true, error: true }; + } + if (protocol === "openai-chat") { + const choices = Array.isArray(object.choices) ? object.choices : []; + const ready = choices.some((choice) => { + if (!choice || typeof choice !== "object") return false; + const choiceObject = choice as Record; + const delta = choiceObject.delta; + return hasContent(delta) || hasContent(choiceObject.message); + }); + return { ready, terminal: false, error: false }; + } + if (protocol === "openai-responses") { + if (object.type === "response.completed" || object.type === "response.done") { + return { ready: false, terminal: true, error: false }; + } + return { + ready: + (object.type === "response.output_text.delta" && hasContent(object.delta)) || + (object.type === "response.function_call_arguments.delta" && hasContent(object.delta)) || + (object.type === "response.output_item.added" && hasContent(object.item)), + terminal: false, + error: false, + }; + } + if (protocol === "gemini") { + const candidates = Array.isArray(object.candidates) ? object.candidates : []; + return { + ready: candidates.some((candidate) => hasContent(candidate)), + terminal: false, + error: false, + }; + } + // Anthropic SSE data events: message_start/message_delta are metadata; a + // content_block_delta or tool use is the first deliverable event. + if ( + object.type === "message_start" || + object.type === "message_delta" || + object.type === "ping" + ) { + return { ready: false, terminal: false, error: false }; + } + if (object.type === "message_stop") { + return { ready: false, terminal: true, error: false }; + } + return { + ready: + (object.type === "content_block_delta" && hasContent(object.delta)) || + (object.type === "content_block_start" && hasContent(object.content_block)) || + hasContent(object.content), + terminal: false, + error: false, + }; +} + +export function classifyDiscoveryChunk( + chunk: Uint8Array | string, + protocol: DiscoveryProtocol +): DiscoveryValidity { + const text = typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk); + if (!text.trim() || text.trim().startsWith(":")) + return { ready: false, terminal: false, error: false }; + if (text.includes("[DONE]")) return { ready: false, terminal: true, error: false }; + + const lines = text.split(/\r?\n/); + let sawTerminal = false; + let sawError = false; + let sawReady = false; + for (const line of lines) { + const candidate = line.startsWith("data:") ? line.slice(5).trim() : line.trim(); + if (!candidate || candidate.startsWith(":")) continue; + try { + const result = classifyJson(JSON.parse(candidate), protocol); + sawTerminal ||= result.terminal; + sawError ||= result.error; + sawReady ||= result.ready; + } catch { + // A raw JSON response may arrive in a single chunk. Plain text is not + // a protocol-safe winner; keep waiting for a parseable event. + } + } + return { ready: sawReady && !sawError, terminal: sawTerminal, error: sawError }; +} + +export class DiscoveryValidityParser { + private buffered = ""; + private readonly decoder = new TextDecoder(); + private _ready = false; + private _terminal = false; + private _error = false; + + constructor(readonly protocol: DiscoveryProtocol) {} + + push(chunk: Uint8Array | string): DiscoveryValidity { + this.buffered += + typeof chunk === "string" ? chunk : this.decoder.decode(chunk, { stream: true }); + const result = classifyDiscoveryChunk(this.buffered, this.protocol); + this._ready ||= result.ready; + this._terminal ||= result.terminal; + this._error ||= result.error; + return { ready: this._ready && !this._error, terminal: this._terminal, error: this._error }; + } + + get ready(): boolean { + return this._ready && !this._error; + } + get terminal(): boolean { + return this._terminal; + } + get error(): boolean { + return this._error; + } +} diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 632807ef2..05cff7803 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -63,6 +63,8 @@ import { buildProxyUrl } from "../url"; import { rectifyBillingHeader } from "./billing-header-rectifier"; import { bindClientAbortListener } from "./client-abort-listener"; import { deriveClientSafeUpstreamErrorMessage } from "./client-error-message"; +import { DiscoveryCoordinator } from "./discovery-coordinator"; +import { type DiscoveryProtocol, DiscoveryValidityParser } from "./discovery-validity"; import { isStandardProxyEndpointPath } from "./endpoint-family-catalog"; import { resolveEndpointPolicy, shouldEnforceStrictEndpointPoolPolicy } from "./endpoint-policy"; import { @@ -1197,6 +1199,12 @@ export class ProxyForwarder { throw new Error("代理上下文缺少供应商或鉴权信息"); } + if (await ProxyForwarder.shouldUseStreamingDiscovery(session)) { + const discoveryPromise = ProxyForwarder.sendStreamingWithDiscovery(session); + void discoveryPromise.catch(() => undefined); + return await discoveryPromise; + } + if (ProxyForwarder.shouldUseStreamingHedge(session)) { const hedgePromise = ProxyForwarder.sendStreamingWithHedge(session); void hedgePromise.catch(() => undefined); @@ -2392,7 +2400,8 @@ export class ProxyForwarder { baseUrl: string, endpointAudit?: { endpointId: number | null; endpointUrl: string }, attemptNumber?: number, - deferDetailSnapshotPersistence: boolean = false + deferDetailSnapshotPersistence: boolean = false, + externalAbortSignal?: AbortSignal ): Promise { if (!provider) { throw new Error("Provider is required"); @@ -3029,12 +3038,16 @@ export class ProxyForwarder { const clientSignal = session.clientAbortSignal; if (clientSignal) abortTransportFrom(clientSignal); }); + const cleanupExternalTransportSignal = bindClientAbortListener(externalAbortSignal, () => { + if (externalAbortSignal) abortTransportFrom(externalAbortSignal); + }); const cleanupCombinedSignal = () => { cleanupResponseTransportSignal(); cleanupClientTransportSignal(); + cleanupExternalTransportSignal(); }; logger.debug("ProxyForwarder: Combined abort signals", { - signalCount: session.clientAbortSignal ? 2 : 1, + signalCount: (session.clientAbortSignal ? 1 : 0) + (externalAbortSignal ? 1 : 0) + 1, }); const init: UndiciFetchOptions = { @@ -3798,6 +3811,13 @@ export class ProxyForwarder { private static shouldUseStreamingHedge(session: ProxySession): boolean { const endpointPolicy = ProxyForwarder.getEndpointPolicy(session); + const routing = session as ProxySession & { + routingMode?: string; + disableStreamingHedge?: boolean; + }; + if (routing.routingMode === "lease_conflict_single" || routing.disableStreamingHedge === true) { + return false; + } return ( (endpointPolicy?.allowRetry ?? true) && (endpointPolicy?.allowProviderSwitch ?? true) && @@ -3806,6 +3826,47 @@ export class ProxyForwarder { ); } + private static async shouldUseStreamingDiscovery(session: ProxySession): Promise { + const settings = await getCachedSystemSettings(); + if (SessionManager.getVersionedBindingCapabilityState() !== "available") { + return false; + } + const endpointPolicy = ProxyForwarder.getEndpointPolicy(session); + const protocol = ProxyForwarder.discoveryProtocol(session); + const message = session.request.message as Record; + const routing = session as ProxySession & { + routingMode?: string; + disableStreamingHedge?: boolean; + }; + return ( + settings.discoveryEnabled === true && + endpointPolicy.allowRetry && + endpointPolicy.allowProviderSwitch && + message.stream === true && + !endpointPolicy.bypassForwarderPreprocessing && + protocol !== "unknown" && + routing.routingMode !== "lease_conflict_single" && + routing.disableStreamingHedge !== true && + !session.isRawCrossProviderFallbackEnabled() + ); + } + + private static discoveryProtocol(session: ProxySession): DiscoveryProtocol { + switch (session.originalFormat) { + case "claude": + return "anthropic"; + case "openai": + return "openai-chat"; + case "response": + return "openai-responses"; + case "gemini": + case "gemini-cli": + return "gemini"; + default: + return "unknown"; + } + } + private static getEndpointPolicy(session: ProxySession) { const policySession = session as unknown as { getEndpointPolicy?: (() => ReturnType) | undefined; @@ -4766,6 +4827,471 @@ export class ProxyForwarder { } } + /** + * Bounded Discovery path. It intentionally lives beside legacy Hedge so the + * existing loser billing and retry semantics remain unchanged while the + * feature is rolled out behind discoveryEnabled. + */ + private static async sendStreamingWithDiscovery(session: ProxySession): Promise { + const initialProvider = session.provider; + if (!initialProvider) throw new Error("代理上下文缺少供应商"); + const settings = await getCachedSystemSettings(); + const concurrency = Math.max(1, Math.floor(settings.discoveryConcurrency ?? 2)); + const maxRounds = Math.max(1, Math.floor(settings.maxDiscoveryRounds ?? 2)); + const discoverySlaMs = Math.max(1, settings.discoverySlaMs ?? 10_000); + const stickySlaMs = Math.max(discoverySlaMs, settings.stickySlaMs ?? 20_000); + const totalTimeoutMs = Math.max(stickySlaMs, settings.racingTotalTimeoutMs ?? 60_000); + const protocol = ProxyForwarder.discoveryProtocol(session); + const coordinator = new DiscoveryCoordinator({ concurrency, maxRounds }); + const bindingKeyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + // Provider selection normally populates this snapshot for a reused Sticky. + // For a cold start, initialize it once so finalization can use generation + // CAS without another read during winner commit or timeout cleanup. + let bindingSnapshot = session.getSessionBindingSnapshot(); + if (!bindingSnapshot && session.sessionId && bindingKeyId != null) { + const binding = await SessionManager.getSessionBindingSnapshot( + session.sessionId, + bindingKeyId + ); + if (binding.status === "ok") { + bindingSnapshot = binding.snapshot; + session.setSessionBindingSnapshot(binding.snapshot); + } + } + const attempts = new Map< + string, + StreamingHedgeAttempt & { + id: string; + kind: "normal" | "fallback"; + controller: AbortController; + parser: DiscoveryValidityParser; + chunks: Uint8Array[]; + pending: boolean; + ready: boolean; + round: number; + } + >(); + const launched = new Set(); + let sequence = 0; + let currentRound = 1; + let winner: (typeof attempts extends Map ? V : never) | null = null; + let committed = false; + let settled = false; + let noMoreCandidates = false; + let lastError: Error | null = null; + let lastErrorCategory: ErrorCategory | null = null; + let totalTimer: NodeJS.Timeout | null = null; + let roundTimer: NodeJS.Timeout | null = null; + let stickyTimer: NodeJS.Timeout | null = null; + let resolveResult: ((result: { response?: Response; error?: Error }) => void) | null = null; + const resultPromise = new Promise<{ response?: Response; error?: Error }>((resolve) => { + resolveResult = resolve; + }); + + const releaseProviderRef = (providerId: number) => { + if (!session.sessionId) return; + const consumer = (session as { consumeProviderSessionRef?: (id: number) => boolean }) + .consumeProviderSessionRef; + if (consumer?.call(session, providerId)) { + void RateLimitService.releaseProviderSession(providerId, session.sessionId); + } + }; + + const cancelAttempt = (attempt: (typeof winner & { id: string }) | null, reason: string) => { + if (!attempt?.pending) return; + attempt.pending = false; + try { + attempt.controller.abort(new Error(reason)); + } catch { + /* abort is best effort */ + } + void attempt.reader?.cancel(reason).catch(() => undefined); + try { + attempt.releaseAgent?.(); + } catch { + /* release is idempotent */ + } + releaseProviderRef(attempt.provider.id); + }; + + const cancelLosers = (keep: typeof winner = null) => { + for (const attempt of attempts.values()) { + if (attempt !== keep) cancelAttempt(attempt, "discovery_loser"); + } + }; + + const settleFailure = async (error: Error) => { + if (settled) return; + settled = true; + if (totalTimer) clearTimeout(totalTimer); + if (roundTimer) clearTimeout(roundTimer); + if (stickyTimer) clearTimeout(stickyTimer); + cancelLosers(); + const attempted = new Set(launched); + await ProxyForwarder.clearSessionProviderBindings(session, attempted); + resolveResult?.({ error }); + }; + + const commit = async (attempt: typeof winner) => { + if (!attempt || committed || settled || !attempt.response || !attempt.reader) return; + committed = true; + winner = attempt; + attempt.pending = false; + if (totalTimer) clearTimeout(totalTimer); + if (roundTimer) clearTimeout(roundTimer); + if (stickyTimer) clearTimeout(stickyTimer); + cancelLosers(attempt); + session.setProvider(attempt.provider); + if (attempt.session !== session) + ProxyForwarder.syncWinningAttemptSession(session, attempt.session); + + setDeferredStreamingFinalization(session, { + providerId: attempt.provider.id, + providerName: attempt.provider.name, + providerPriority: attempt.provider.priority || 0, + attemptNumber: attempt.sequence, + totalProvidersAttempted: launched.size, + isFirstAttempt: attempt.provider.id === initialProvider.id, + isFailoverSuccess: attempt.provider.id !== initialProvider.id, + endpointId: attempt.endpointAudit.endpointId, + endpointUrl: attempt.endpointAudit.endpointUrl, + upstreamStatusCode: attempt.response.status, + isHedgeWinner: false, + billHedgeLosers: false, + bindingIntent: + attempt.kind === "fallback" + ? "none" + : bindingSnapshot?.providerId == null + ? "create" + : "renew", + bindingSnapshot, + requiresCompletionMarker: attempt.kind !== "fallback", + }); + const prefix = + attempt.chunks.length === 1 + ? attempt.chunks[0] + : (() => { + const size = attempt.chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0); + const output = new Uint8Array(size); + let offset = 0; + for (const chunk of attempt.chunks) { + output.set(chunk, offset); + offset += chunk.byteLength; + } + return output; + })(); + resolveResult?.({ + response: new Response( + ProxyForwarder.buildBufferedFirstChunkStream(prefix, attempt.reader), + { + status: attempt.response.status, + statusText: attempt.response.statusText, + headers: attempt.response.headers, + } + ), + }); + }; + + const chooseCandidate = async (): Promise => { + const candidates = await ProxyProviderResolver.pickDiscoveryProviders( + session, + 1, + Array.from(launched) + ); + if (!candidates[0]) { + noMoreCandidates = true; + return null; + } + return candidates[0]; + }; + + const launch = async (provider: Provider, kind: "normal" | "fallback"): Promise => { + if (settled || committed || launched.has(provider.id)) return; + launched.add(provider.id); + if (provider.id !== initialProvider.id && session.sessionId) { + const limit = provider.limitConcurrentSessions || 0; + const check = await RateLimitService.checkAndTrackProviderSession( + provider.id, + session.sessionId, + limit + ); + if (!check.allowed) { + launched.delete(provider.id); + throw new ProxyError(check.reason || "Provider concurrent limit reached", 503); + } + if (check.referenced) session.recordProviderSessionRef(provider.id); + } + const endpoint = await ProxyForwarder.resolveStreamingHedgeEndpoint(session, provider); + const attemptSession = + provider.id === initialProvider.id + ? session + : ProxyForwarder.createStreamingShadowSession(session, provider); + attemptSession.setProvider(provider); + const controller = new AbortController(); + const id = `${provider.id}:${sequence + 1}`; + const attempt = { + id, + kind, + controller, + parser: new DiscoveryValidityParser(protocol), + chunks: [], + pending: true, + ready: false, + round: currentRound, + provider, + session: attemptSession, + baseUrl: endpoint.baseUrl, + endpointAudit: { endpointId: endpoint.endpointId, endpointUrl: endpoint.endpointUrl }, + modelRedirect: undefined, + responseController: null, + clearResponseTimeout: null, + firstByteTimeoutMs: 0, + sequence: ++sequence, + requestAttemptCount: 1, + reactiveRectifierRetryState: { + thinkingSignatureRetried: false, + thinkingBudgetRetried: false, + thinkingEffortConflictRetried: false, + geminiFunctionIdRetried: false, + }, + settled: false, + thresholdTriggered: false, + thresholdTimer: null, + reader: null, + response: null, + releaseAgent: null, + agentReleased: false, + billAsLoser: false, + loserBillingStarted: false, + firstChunk: null, + billingSnapshot: null, + } as typeof winner & { + id: string; + kind: "normal" | "fallback"; + controller: AbortController; + parser: DiscoveryValidityParser; + chunks: Uint8Array[]; + pending: boolean; + ready: boolean; + round: number; + }; + attempts.set(id, attempt); + coordinator.addAttempt({ + id, + providerId: provider.id, + priority: provider.priority || 0, + kind, + ready: false, + pending: true, + round: currentRound, + launchOrder: attempt.sequence, + }); + + void ProxyForwarder.doForward( + attempt.session, + { ...provider, firstByteTimeoutStreamingMs: 0 }, + endpoint.baseUrl, + attempt.endpointAudit, + attempt.requestAttemptCount, + true, + controller.signal + ) + .then(async (response) => { + const runtime = attempt.session as ProxySessionWithAttemptRuntime; + attempt.responseController = runtime.responseController ?? null; + attempt.clearResponseTimeout = runtime.clearResponseTimeout ?? null; + attempt.releaseAgent = runtime.releaseAgent ?? null; + attempt.clearResponseTimeout?.(); + attempt.response = response; + if (!response.body) + throw new EmptyResponseError(provider.id, provider.name, "empty_body"); + attempt.reader = response.body.getReader(); + while (!committed && !settled && attempt.pending) { + const item = await attempt.reader.read(); + if (item.done) throw new EmptyResponseError(provider.id, provider.name, "empty_body"); + if (!item.value || item.value.byteLength === 0) continue; + attempt.chunks.push(item.value); + const validity = attempt.parser.push(item.value); + if (validity.error || validity.terminal) + throw new ProxyError("Invalid upstream discovery response", 502); + if (!validity.ready) continue; + attempt.ready = true; + const normalPendingHigher = Array.from(attempts.values()).some( + (other) => + other.pending && + other.kind === "normal" && + (other.provider.priority || 0) < (provider.priority || 0) + ); + if (normalPendingHigher) continue; + const action = coordinator.markReady(id); + if (action.type === "commit_normal" || action.type === "promote_fallback") + await commit(attempt); + return; + } + }) + .catch(async (error) => { + if (committed || settled || !attempt.pending) return; + attempt.pending = false; + coordinator.markFailed(id); + lastError = error instanceof Error ? error : new Error(String(error)); + lastErrorCategory = await categorizeErrorAsync(lastError); + session.addProviderToChain(provider, { + ...attempt.endpointAudit, + reason: "retry_failed", + attemptNumber: attempt.sequence, + statusCode: lastError instanceof ProxyError ? lastError.statusCode : undefined, + errorMessage: lastError.message, + }); + if ( + lastErrorCategory === ErrorCategory.PROVIDER_ERROR && + !(lastError instanceof ProxyError && lastError.statusCode === 404) + ) { + await recordFailure(provider.id, lastError).catch(() => undefined); + } + attempt.releaseAgent?.(); + releaseProviderRef(provider.id); + const replacement = await chooseCandidate(); + if (replacement && !committed && !settled) await launch(replacement, "normal"); + if ( + Array.from(attempts.values()).every((candidate) => !candidate.pending) && + noMoreCandidates + ) { + await settleFailure( + ProxyForwarder.resolveHedgeTerminalError(lastError, lastErrorCategory) + ); + } + }); + }; + + const launchNextRound = async () => { + if (settled || committed) return; + currentRound += 1; + if (currentRound > maxRounds) return; + coordinator.beginRound(); + const candidate = await chooseCandidate(); + if (candidate) { + try { + await launch(candidate, "normal"); + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + noMoreCandidates = true; + } + } + if (!committed && !settled) { + roundTimer = setTimeout(() => void onBoundary(), discoverySlaMs); + } + }; + + const onBoundary = async () => { + if (settled || committed) return; + const fallback = Array.from(attempts.values()).find( + (attempt) => attempt.pending && attempt.kind === "fallback" + ); + const pendingNormals = Array.from(attempts.values()) + .filter((attempt) => attempt.pending && attempt.kind === "normal") + .sort( + (a, b) => + (a.provider.priority || 0) - (b.provider.priority || 0) || a.sequence - b.sequence + ); + const readyNormal = pendingNormals.find((attempt) => attempt.ready); + if (readyNormal) { + await commit(readyNormal); + return; + } + // The fallback is held during the SLA window, but at the round + // boundary it may take over when no normal result is ready. + if (fallback?.ready) { + await commit(fallback); + return; + } + if (pendingNormals[0]) { + if (fallback) { + for (const loser of pendingNormals) cancelAttempt(loser, "discovery_round_boundary"); + if (currentRound < maxRounds) await launchNextRound(); + return; + } + pendingNormals[0].kind = "fallback"; + for (const loser of pendingNormals.slice(1)) + cancelAttempt(loser, "discovery_round_boundary"); + if (currentRound < maxRounds) await launchNextRound(); + return; + } + if (currentRound < maxRounds) await launchNextRound(); + else await settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); + }; + + const cleanupAbort = bindClientAbortListener(session.clientAbortSignal, () => { + if (settled || committed) return; + void settleFailure(new ProxyError("Request aborted by client", 499, undefined, true)); + }); + + totalTimer = setTimeout(() => { + if (settled || committed) return; + const fallback = Array.from(attempts.values()).find( + (attempt) => attempt.pending && attempt.kind === "fallback" && attempt.ready + ); + if (fallback) void commit(fallback); + else void settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); + }, totalTimeoutMs); + + try { + const hasSticky = session.shouldReuseProvider() && !!session.sessionId; + try { + await launch(initialProvider, "normal"); + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + const replacement = await chooseCandidate(); + if (replacement) await launch(replacement, "normal"); + else await settleFailure(ProxyForwarder.resolveHedgeTerminalError(lastError, null)); + } + const initial = hasSticky ? concurrency - 1 : Math.max(0, concurrency - 1); + if (hasSticky) { + stickyTimer = setTimeout(() => { + const sticky = Array.from(attempts.values()).find( + (attempt) => attempt.pending && attempt.provider.id === initialProvider.id + ); + if (sticky) { + sticky.kind = "fallback"; + if (bindingSnapshot && bindingSnapshot.providerId === initialProvider.id) { + void SessionManager.clearVersionedSessionProvider( + bindingSnapshot, + initialProvider.id, + Math.ceil((settings.stickyTimeoutCooldownMs ?? 300_000) / 1000) + ).catch((error) => + logger.debug("[Discovery] Failed to clear timed-out Sticky", { error }) + ); + } + if (currentRound < maxRounds) void launchNextRound(); + else void onBoundary(); + } + }, stickySlaMs); + } else { + const candidates = await ProxyProviderResolver.pickDiscoveryProviders( + session, + initial, + Array.from(launched) + ); + for (const provider of candidates) { + try { + await launch(provider, "normal"); + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + noMoreCandidates = true; + } + } + roundTimer = setTimeout(() => void onBoundary(), discoverySlaMs); + } + const result = await resultPromise; + if (result.error) throw result.error; + return result.response as Response; + } finally { + cleanupAbort(); + if (totalTimer) clearTimeout(totalTimer); + if (roundTimer) clearTimeout(roundTimer); + if (stickyTimer) clearTimeout(stickyTimer); + } + } + private static async resolveStreamingHedgeEndpoint( session: ProxySession, provider: Provider diff --git a/src/app/v1/_lib/proxy/provider-selector.ts b/src/app/v1/_lib/proxy/provider-selector.ts index 10c397000..ee290e1aa 100644 --- a/src/app/v1/_lib/proxy/provider-selector.ts +++ b/src/app/v1/_lib/proxy/provider-selector.ts @@ -457,6 +457,43 @@ export class ProxyProviderResolver { return provider; } + /** + * Select a bounded Discovery batch using the exact same filters, priority + * and weighted selection as the normal selector. The method is intentionally + * additive: legacy initial selection/fallback keeps its existing behavior. + */ + static async pickDiscoveryProviders( + session: ProxySession, + count: number, + excludeIds: number[] = [] + ): Promise { + const selected: Provider[] = []; + const excluded = new Set(excludeIds); + const limit = Math.max(0, Math.floor(count)); + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + while (selected.length < limit) { + const provider = await ProxyProviderResolver.pickRandomProviderWithExclusion( + session, + Array.from(excluded) + ); + if (!provider || excluded.has(provider.id)) break; + if (session.sessionId && keyId != null) { + const cooldown = await SessionManager.isSessionProviderCoolingDown( + session.sessionId, + keyId, + provider.id + ); + if (cooldown.status === "ok" && cooldown.coolingDown) { + excluded.add(provider.id); + continue; + } + } + selected.push(provider); + excluded.add(provider.id); + } + return selected; + } + /** * 查找可复用的供应商（基于 session） */ @@ -465,9 +502,22 @@ export class ProxyProviderResolver { return null; } - // 从 Redis 读取该 session 绑定的 provider + // Read the binding once and retain its generation for Discovery timeout + // cleanup/finalization. Re-reading here would allow an older request to + // clear a newer binding (ABA). const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; - const providerId = await SessionManager.getSessionProvider(session.sessionId, keyId); + let providerId: number | null = null; + if (keyId != null) { + const binding = await SessionManager.getSessionBindingSnapshot(session.sessionId, keyId); + if (binding.status === "ok") { + session.setSessionBindingSnapshot(binding.snapshot); + providerId = binding.snapshot.providerId; + } else if (binding.legacyFallbackAllowed) { + providerId = await SessionManager.getSessionProvider(session.sessionId, keyId); + } + } else { + providerId = await SessionManager.getSessionProvider(session.sessionId, keyId); + } if (!providerId) { logger.debug("ProviderSelector: Session has no bound provider", { sessionId: session.sessionId, diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index acd5eb26f..bc36a1a39 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -1150,6 +1150,15 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const clearSessionBinding = async () => { if (!session.sessionId) return; const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + if (meta?.bindingIntent === "none") return; + if (meta?.bindingSnapshot && keyId != null) { + await SessionManager.clearVersionedSessionProvider( + meta.bindingSnapshot, + providerIdForPersistence, + 0 + ); + return; + } await SessionManager.clearSessionProvider(session.sessionId, providerIdForPersistence, keyId); }; @@ -1165,6 +1174,10 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const detected = shouldDetectFake200 ? detectUpstreamErrorFromSseOrJsonText(allContent) : ({ isError: false } as const); + const completionMarkerMissing = + meta?.requiresCompletionMarker === true && + streamEndedNormally && + !hasStreamCompletionMarker(allContent); let clientAbortGateUsage: FinalizeDeferredStreamingResult["clientAbortGateUsage"]; const clientAbortCompleteSuccess = (() => { if (!clientAborted || upstreamStatusCode < 200 || upstreamStatusCode >= 300) { @@ -1209,6 +1222,9 @@ function finalizeDeferredStreamingFinalizationIfNeeded( effectiveStatusCode = 502; } errorMessage = detected.detail ? `${detected.code}: ${detected.detail}` : detected.code; + } else if (completionMarkerMissing) { + effectiveStatusCode = 502; + errorMessage = "STREAM_COMPLETION_MARKER_MISSING"; } else if (clientAbortCompleteSuccess) { effectiveStatusCode = upstreamStatusCode; errorMessage = null; @@ -1239,6 +1255,7 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const shouldClearSessionBindingOnFailure = ((clientAborted || !streamEndedNormally) && !clientAbortCompleteSuccess) || detected.isError || + completionMarkerMissing || (upstreamStatusCode >= 400 && errorMessage !== null); // 未启用延迟结算 / provider 缺失： @@ -1326,6 +1343,43 @@ function finalizeDeferredStreamingFinalizationIfNeeded( }; } + if (completionMarkerMissing) { + session.addProviderToChain(providerForChain, { + endpointId: meta.endpointId, + endpointUrl: meta.endpointUrl, + reason: "retry_failed", + attemptNumber: meta.attemptNumber, + statusCode: effectiveStatusCode, + errorMessage: errorMessage ?? undefined, + }); + + const commitSideEffects = async () => { + await clearSessionBinding(); + if (session.getEndpointPolicy().allowCircuitBreakerAccounting) { + try { + const { recordFailure } = await import("@/lib/circuit-breaker"); + await recordFailure(meta.providerId, new Error(errorMessage ?? "STREAM_ABORTED")); + } catch (cbError) { + logger.warn("[ResponseHandler] Failed to record missing stream completion marker", { + providerId: meta.providerId, + sessionId: session.sessionId ?? null, + error: cbError, + }); + } + } + }; + + return { + effectiveStatusCode, + errorMessage, + providerIdForPersistence, + isHedgeWinner, + billHedgeLosers, + clientAbortGateUsage, + commitSideEffects, + }; + } + if (detected.isError) { logger.warn("[ResponseHandler] SSE completed but body indicates error (fake 200)", { providerId: meta.providerId, @@ -1487,15 +1541,26 @@ function finalizeDeferredStreamingFinalizationIfNeeded( } // Hedge winner: commitWinner() already performed session binding and chain logging. - if (!meta.isHedgeWinner && session.sessionId) { - const result = await SessionManager.updateSessionBindingSmart( - session.sessionId, - meta.providerId, - meta.providerPriority, - meta.isFirstAttempt, - meta.isFailoverSuccess, - session.authState?.key?.id ?? session.messageContext?.key?.id ?? null - ); + if (meta.bindingIntent !== "none" && !meta.isHedgeWinner && session.sessionId) { + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + const result = + meta.bindingSnapshot && keyId != null + ? await SessionManager.compareAndSetSessionProvider( + meta.bindingSnapshot, + meta.providerId + ).then((cas) => ({ + updated: cas.status === "ok", + reason: cas.status === "ok" ? "discovery_generation_cas" : cas.reason, + details: cas.status, + })) + : await SessionManager.updateSessionBindingSmart( + session.sessionId, + meta.providerId, + meta.providerPriority, + meta.isFirstAttempt, + meta.isFailoverSuccess, + keyId + ); if (result.updated) { logger.info("[ResponseHandler] Session binding updated (stream finalized)", { diff --git a/src/app/v1/_lib/proxy/session.ts b/src/app/v1/_lib/proxy/session.ts index 10d6b4740..18ff8bcae 100644 --- a/src/app/v1/_lib/proxy/session.ts +++ b/src/app/v1/_lib/proxy/session.ts @@ -1,6 +1,7 @@ import type { Context } from "hono"; import { logger } from "@/lib/logger"; import { writeLiveChain } from "@/lib/redis/live-chain-store"; +import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import { clientRequestsContext1m as clientRequestsContext1mHelper } from "@/lib/special-attributes"; import { type ResolvedPricing, @@ -206,6 +207,11 @@ export class ProxySession { // 失败切换 provider 时只能释放这里记录过的引用，避免 hedge/fallback 释放未 acquire 的 Redis 计数。 private providerSessionRefs = new Set(); + // Snapshot captured during provider selection. Discovery reuses this exact + // generation for timeout cleanup/finalization instead of performing a + // second read that could race with another request's binding update. + private sessionBindingSnapshot: SessionBindingSnapshot | null = null; + private constructor(init: { startTime: number; method: string; @@ -349,6 +355,14 @@ export class ProxySession { } } + setSessionBindingSnapshot(snapshot: SessionBindingSnapshot | null): void { + this.sessionBindingSnapshot = snapshot; + } + + getSessionBindingSnapshot(): SessionBindingSnapshot | null { + return this.sessionBindingSnapshot; + } + recordProviderSessionRef(providerId: number): void { if (!this.providerSessionRefs) { this.providerSessionRefs = new Set(); diff --git a/src/app/v1/_lib/proxy/stream-finalization.ts b/src/app/v1/_lib/proxy/stream-finalization.ts index 0f989bb3b..497207f1f 100644 --- a/src/app/v1/_lib/proxy/stream-finalization.ts +++ b/src/app/v1/_lib/proxy/stream-finalization.ts @@ -1,3 +1,4 @@ +import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import type { ProxySession } from "./session"; /** @@ -35,6 +36,11 @@ export type DeferredStreamingFinalization = { * coexists with asynchronously accumulated loser costs without clobbering. */ billHedgeLosers?: boolean; + /** Discovery delays binding until the stream has a valid completion marker. */ + bindingIntent?: "create" | "renew" | "none"; + bindingSnapshot?: SessionBindingSnapshot | null; + /** Discovery winners must satisfy the protocol completion marker before binding. */ + requiresCompletionMarker?: boolean; }; const deferredMeta = new WeakMap(); diff --git a/src/lib/config/system-settings-cache.ts b/src/lib/config/system-settings-cache.ts index 736b55238..6649603f5 100644 --- a/src/lib/config/system-settings-cache.ts +++ b/src/lib/config/system-settings-cache.ts @@ -166,6 +166,13 @@ export async function getCachedSystemSettings(): Promise { publicStatusWindowHours: DEFAULT_SETTINGS.publicStatusWindowHours, publicStatusAggregationIntervalMinutes: DEFAULT_SETTINGS.publicStatusAggregationIntervalMinutes, + discoveryEnabled: false, + discoveryConcurrency: 2, + maxDiscoveryRounds: 2, + discoverySlaMs: 10_000, + stickySlaMs: 20_000, + racingTotalTimeoutMs: 60_000, + stickyTimeoutCooldownMs: 300_000, quotaDbRefreshIntervalSeconds: 10, quotaLeasePercent5h: 0.05, quotaLeasePercentDaily: 0.05, diff --git a/src/types/system-config.ts b/src/types/system-config.ts index 3be265c11..16ef36a98 100644 --- a/src/types/system-config.ts +++ b/src/types/system-config.ts @@ -146,6 +146,15 @@ export interface SystemSettings { publicStatusWindowHours: number; publicStatusAggregationIntervalMinutes: number; + /** Bounded streaming Discovery (PR2; persisted/configured in PR3). */ + discoveryEnabled?: boolean; + discoveryConcurrency?: number; + maxDiscoveryRounds?: number; + discoverySlaMs?: number; + stickySlaMs?: number; + racingTotalTimeoutMs?: number; + stickyTimeoutCooldownMs?: number; + createdAt: Date; updatedAt: Date; } @@ -170,6 +179,14 @@ export interface UpdateSystemSettingsInput { // 供应商竞速输家计费（可选） billHedgeLosers?: boolean; + discoveryEnabled?: boolean; + discoveryConcurrency?: number; + maxDiscoveryRounds?: number; + discoverySlaMs?: number; + stickySlaMs?: number; + racingTotalTimeoutMs?: number; + stickyTimeoutCooldownMs?: number; + // 系统时区配置（可选） timezone?: string | null; diff --git a/tests/unit/proxy/discovery-coordinator.test.ts b/tests/unit/proxy/discovery-coordinator.test.ts new file mode 100644 index 000000000..70b09b6d0 --- /dev/null +++ b/tests/unit/proxy/discovery-coordinator.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; +import { DiscoveryCoordinator } from "@/app/v1/_lib/proxy/discovery-coordinator"; + +const attempt = (id: string, priority: number, kind: "normal" | "fallback" = "normal") => ({ + id, + providerId: Number(id.replace(/\D/g, "")) || 1, + priority, + kind, + ready: false, + pending: true, + round: 1, + launchOrder: Number(id.replace(/\D/g, "")) || 1, +}); + +describe("DiscoveryCoordinator", () => { + it("commits the highest priority ready normal attempt", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("a", 10)); + coordinator.addAttempt(attempt("b", 1)); + expect(coordinator.markReady("a")).toEqual({ type: "none" }); + expect(coordinator.markReady("b")).toEqual({ type: "commit_normal", attemptId: "b" }); + expect(coordinator.state).toBe("WINNER_COMMITTED"); + }); + + it("promotes one pending normal to fallback at a round boundary", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("a", 1)); + coordinator.addAttempt(attempt("b", 2)); + const action = coordinator.onRoundBoundary(); + expect(action.type).toBe("cancel"); + expect(coordinator.snapshot.find((item) => item.id === "a")?.kind).toBe("fallback"); + expect(coordinator.snapshot.filter((item) => item.pending)).toHaveLength(1); + }); + + it("ignores callbacks from an old request epoch", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("a", 1)); + const epoch = coordinator.epochs; + coordinator.cancelRequest(); + expect(coordinator.markReady("a", epoch.requestEpoch, epoch.roundEpoch)).toEqual({ + type: "none", + }); + }); + + it("promotes a ready fallback at the round boundary when no normal is ready", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("a", 1, "fallback")); + coordinator.addAttempt(attempt("b", 1, "normal")); + expect(coordinator.markReady("a")).toEqual({ type: "none" }); + expect(coordinator.onRoundBoundary()).toEqual({ type: "promote_fallback", attemptId: "a" }); + expect(coordinator.snapshot.find((item) => item.id === "a")?.kind).toBe("fallback"); + }); + + it("chooses the best ready normal at a boundary", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 3, maxRounds: 1 }); + coordinator.addAttempt(attempt("a", 10)); + coordinator.addAttempt(attempt("b", 1)); + coordinator.addAttempt(attempt("c", 5)); + coordinator.markReady("a"); + coordinator.markReady("c"); + expect(coordinator.onRoundBoundary()).toEqual({ type: "commit_normal", attemptId: "c" }); + }); +}); diff --git a/tests/unit/proxy/discovery-validity.test.ts b/tests/unit/proxy/discovery-validity.test.ts new file mode 100644 index 000000000..4d5e6cae2 --- /dev/null +++ b/tests/unit/proxy/discovery-validity.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; +import { + DiscoveryValidityParser, + classifyDiscoveryChunk, +} from "@/app/v1/_lib/proxy/discovery-validity"; + +describe("discovery validity", () => { + it("does not treat Anthropic metadata as a winner", () => { + expect(classifyDiscoveryChunk('data: {"type":"message_start"}\n\n', "anthropic").ready).toBe( + false + ); + expect( + classifyDiscoveryChunk( + 'data: {"type":"content_block_delta","delta":{"text":"hi"}}\n\n', + "anthropic" + ).ready + ).toBe(true); + }); + + it("accepts OpenAI Chat delta and rejects DONE", () => { + expect( + classifyDiscoveryChunk('data: {"choices":[{"delta":{"content":"hi"}}]}\n', "openai-chat") + .ready + ).toBe(true); + expect(classifyDiscoveryChunk("data: [DONE]\n", "openai-chat").terminal).toBe(true); + }); + + it("rejects errors even when a later chunk contains content", () => { + const parser = new DiscoveryValidityParser("openai-responses"); + expect(parser.push('{"type":"response.failed","error":{"message":"no"}}').error).toBe(true); + expect(parser.push('{"type":"response.output_text.delta","delta":"late"}').ready).toBe(false); + }); + + it("does not promote empty tool or content events", () => { + expect( + classifyDiscoveryChunk( + 'data: {"type":"content_block_start","content_block":{"type":"text","text":""}}\n', + "anthropic" + ).ready + ).toBe(false); + expect( + classifyDiscoveryChunk( + 'data: {"choices":[{"delta":{"tool_calls":[{"function":{}}]}}]}\n', + "openai-chat" + ).ready + ).toBe(false); + expect( + classifyDiscoveryChunk( + 'data: {"type":"response.output_text.delta","delta":" "}\n', + "openai-responses" + ).ready + ).toBe(false); + }); + + it("accepts a non-empty function call delta as deliverable content", () => { + expect( + classifyDiscoveryChunk( + 'data: {"type":"response.function_call_arguments.delta","delta":"{\\"x\\":1}"}\n', + "openai-responses" + ).ready + ).toBe(true); + }); +}); From 31c4324d06606dba7815c34421bfa7cf2302b1b0 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 14:25:58 -0400 Subject: [PATCH 05/89] fix(proxy): keep discovery disabled path compatible --- src/app/v1/_lib/proxy/forwarder.ts | 8 +++++++- tests/integration/proxy-hedge-lifecycle.test.ts | 6 ++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 05cff7803..122e00d71 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -3828,7 +3828,13 @@ export class ProxyForwarder { private static async shouldUseStreamingDiscovery(session: ProxySession): Promise { const settings = await getCachedSystemSettings(); - if (SessionManager.getVersionedBindingCapabilityState() !== "available") { + if (settings.discoveryEnabled !== true) { + return false; + } + if ( + typeof SessionManager.getVersionedBindingCapabilityState !== "function" || + SessionManager.getVersionedBindingCapabilityState() !== "available" + ) { return false; } const endpointPolicy = ProxyForwarder.getEndpointPolicy(session); diff --git a/tests/integration/proxy-hedge-lifecycle.test.ts b/tests/integration/proxy-hedge-lifecycle.test.ts index ec3e9a533..390727090 100644 --- a/tests/integration/proxy-hedge-lifecycle.test.ts +++ b/tests/integration/proxy-hedge-lifecycle.test.ts @@ -518,7 +518,8 @@ describe("proxy hedge transport/lifecycle integration (persistence and control-p outputTokens: 3, providerId: 2, statusCode: 200, - }) + }), + expect.any(Object) ); expect(state.updateMessageRequestDetailsIfUnfinalized).not.toHaveBeenCalled(); @@ -615,7 +616,8 @@ describe("proxy hedge transport/lifecycle integration (persistence and control-p expect(state.durableTerminal).toHaveBeenCalledOnce(); expect(state.durableTerminal).toHaveBeenCalledWith( MESSAGE.id, - expect.objectContaining({ statusCode: 502 }) + expect.objectContaining({ statusCode: 502 }), + expect.any(Object) ); expect(agents.release).toHaveBeenCalledOnce(); expect(agents.pool.getPoolStats().activeRequests).toBe(0); From c17a75646d80e4577100cdf83cbc878dc95bb08e Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 14:25:58 -0400 Subject: [PATCH 06/89] fix(proxy): keep discovery disabled path compatible --- src/app/v1/_lib/proxy/forwarder.ts | 8 +++++++- tests/integration/proxy-hedge-lifecycle.test.ts | 6 ++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 05cff7803..122e00d71 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -3828,7 +3828,13 @@ export class ProxyForwarder { private static async shouldUseStreamingDiscovery(session: ProxySession): Promise { const settings = await getCachedSystemSettings(); - if (SessionManager.getVersionedBindingCapabilityState() !== "available") { + if (settings.discoveryEnabled !== true) { + return false; + } + if ( + typeof SessionManager.getVersionedBindingCapabilityState !== "function" || + SessionManager.getVersionedBindingCapabilityState() !== "available" + ) { return false; } const endpointPolicy = ProxyForwarder.getEndpointPolicy(session); diff --git a/tests/integration/proxy-hedge-lifecycle.test.ts b/tests/integration/proxy-hedge-lifecycle.test.ts index ec3e9a533..390727090 100644 --- a/tests/integration/proxy-hedge-lifecycle.test.ts +++ b/tests/integration/proxy-hedge-lifecycle.test.ts @@ -518,7 +518,8 @@ describe("proxy hedge transport/lifecycle integration (persistence and control-p outputTokens: 3, providerId: 2, statusCode: 200, - }) + }), + expect.any(Object) ); expect(state.updateMessageRequestDetailsIfUnfinalized).not.toHaveBeenCalled(); @@ -615,7 +616,8 @@ describe("proxy hedge transport/lifecycle integration (persistence and control-p expect(state.durableTerminal).toHaveBeenCalledOnce(); expect(state.durableTerminal).toHaveBeenCalledWith( MESSAGE.id, - expect.objectContaining({ statusCode: 502 }) + expect.objectContaining({ statusCode: 502 }), + expect.any(Object) ); expect(agents.release).toHaveBeenCalledOnce(); expect(agents.pool.getPoolStats().activeRequests).toBe(0); From 626f8042975c8506faacc798c0de95e527bf4f77 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 14:42:22 -0400 Subject: [PATCH 07/89] test(proxy): keep hedge lifecycle assertions current --- tests/integration/proxy-hedge-lifecycle.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/proxy-hedge-lifecycle.test.ts b/tests/integration/proxy-hedge-lifecycle.test.ts index ec3e9a533..390727090 100644 --- a/tests/integration/proxy-hedge-lifecycle.test.ts +++ b/tests/integration/proxy-hedge-lifecycle.test.ts @@ -518,7 +518,8 @@ describe("proxy hedge transport/lifecycle integration (persistence and control-p outputTokens: 3, providerId: 2, statusCode: 200, - }) + }), + expect.any(Object) ); expect(state.updateMessageRequestDetailsIfUnfinalized).not.toHaveBeenCalled(); @@ -615,7 +616,8 @@ describe("proxy hedge transport/lifecycle integration (persistence and control-p expect(state.durableTerminal).toHaveBeenCalledOnce(); expect(state.durableTerminal).toHaveBeenCalledWith( MESSAGE.id, - expect.objectContaining({ statusCode: 502 }) + expect.objectContaining({ statusCode: 502 }), + expect.any(Object) ); expect(agents.release).toHaveBeenCalledOnce(); expect(agents.pool.getPoolStats().activeRequests).toBe(0); From 856a44ec8a7a98ba8f6e70c8b1d2520675cdb667 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 14:48:30 -0400 Subject: [PATCH 08/89] feat(settings): configure bounded streaming discovery --- docs/streaming-discovery.md | 61 + drizzle/0110_daffy_rawhide_kid.sql | 7 + drizzle/meta/0110_snapshot.json | 4691 +++++++++++++++++ drizzle/meta/_journal.json | 9 +- messages/en/settings/config.json | 10 + messages/ja/settings/config.json | 10 + messages/ru/settings/config.json | 10 + messages/zh-CN/settings/config.json | 10 + messages/zh-TW/settings/config.json | 10 + src/actions/system-config.ts | 30 + .../_components/system-settings-form.tsx | 89 + src/app/[locale]/settings/config/page.tsx | 7 + src/app/api/admin/system-config/route.ts | 18 + src/drizzle/schema.ts | 9 + src/lib/api-client/v1/openapi-types.gen.ts | 42 + src/lib/api/v1/schemas/system-config.ts | 19 + src/lib/config/system-settings-cache.ts | 28 +- src/lib/validation/schemas.ts | 455 +- src/repository/_shared/transformers.ts | 7 + src/repository/system-config.ts | 78 + src/types/system-config.ts | 16 +- .../system-config-degradation-ladder.test.ts | 34 +- ...stem-config-update-missing-columns.test.ts | 10 +- .../system-settings-discovery.test.ts | 34 + 24 files changed, 5468 insertions(+), 226 deletions(-) create mode 100644 docs/streaming-discovery.md create mode 100644 drizzle/0110_daffy_rawhide_kid.sql create mode 100644 drizzle/meta/0110_snapshot.json create mode 100644 tests/unit/validation/system-settings-discovery.test.ts diff --git a/docs/streaming-discovery.md b/docs/streaming-discovery.md new file mode 100644 index 000000000..694d560d0 --- /dev/null +++ b/docs/streaming-discovery.md @@ -0,0 +1,61 @@ +# Bounded Streaming Discovery + +Bounded Discovery is an optional cold-start routing mode for streaming +requests. It exists to reduce duplicate upstream spend while preserving a +working request when the first provider is slow. + +## Defaults + +| Setting | Default | Meaning | +| --- | ---: | --- | +| `discoveryEnabled` | `false` | Keep the existing Hedge path until explicitly enabled. | +| `discoveryConcurrency` | `2` | Number of normal providers in the first batch. | +| `maxDiscoveryRounds` | `2` | Maximum Discovery rounds. | +| `discoverySlaMs` | `10000` | First-byte budget for a Discovery round. | +| `stickySlaMs` | `20000` | First-byte budget for an existing Sticky provider. | +| `racingTotalTimeoutMs` | `60000` | Total pre-winner deadline; it is cleared after a winner is committed. | +| `stickyTimeoutCooldownMs` | `300000` | Session/provider cooldown after a Sticky timeout. | + +The total deadline must be at least `stickySlaMs + maxDiscoveryRounds * +discoverySlaMs`. The UI and API reject configurations that do not satisfy +this relationship. + +## Request lifecycle + +- A healthy Sticky provider is probed alone. If it times out, it becomes the + single fallback for this request and receives a cooldown; a later request + may select it again after the cooldown. +- A cold start launches the configured initial normal candidates. The highest + priority ready candidate wins; a lower-priority candidate remains held while + a higher-priority candidate is still inside its SLA window. +- At a round boundary, at most one pending normal attempt is promoted to the + fallback. The next round uses the remaining slots for new normal candidates, + so `discoveryConcurrency=2` means `one fallback + one new candidate`. +- A fallback that has produced a valid prefix is held until the current normal + window closes, all normal candidates fail, or no candidates remain. A normal + winner always has precedence during the window. +- Discovery losers are cancelled and their readers/agents/provider-session + references are released. They do not enter legacy `bill_hedge_losers` + draining. +- Sticky binding is written only after a natural, successful stream completion + with the protocol completion marker and a generation-aware CAS. Fake-200, + incomplete, and client-aborted streams do not create or renew Sticky. + +Discovery is eligible only for supported streaming protocol families and when +the versioned Redis binding capability is available. If Redis capability is +unknown/unavailable, the existing provider selection and Hedge behavior remain +active. + +## Rollout + +1. Apply the system-settings migration. +2. Confirm the Redis versioned-binding capability probe is `available`. +3. Leave `discoveryEnabled=false` while validating the existing Hedge and + versioned binding checks. +4. Enable Discovery for a controlled group, observe provider-chain outcomes, + first-token latency, fallback promotions, cancellations, CAS conflicts, and + final 503s. +5. Disable the setting to return immediately to the legacy Hedge path. + +This feature does not change the final client failure contract: an exhausted +request continues to return the existing `503` mapping. diff --git a/drizzle/0110_daffy_rawhide_kid.sql b/drizzle/0110_daffy_rawhide_kid.sql new file mode 100644 index 000000000..f52f66675 --- /dev/null +++ b/drizzle/0110_daffy_rawhide_kid.sql @@ -0,0 +1,7 @@ +ALTER TABLE "system_settings" ADD COLUMN "discovery_enabled" boolean DEFAULT false NOT NULL;--> statement-breakpoint +ALTER TABLE "system_settings" ADD COLUMN "discovery_concurrency" integer DEFAULT 2 NOT NULL;--> statement-breakpoint +ALTER TABLE "system_settings" ADD COLUMN "max_discovery_rounds" integer DEFAULT 2 NOT NULL;--> statement-breakpoint +ALTER TABLE "system_settings" ADD COLUMN "discovery_sla_ms" integer DEFAULT 10000 NOT NULL;--> statement-breakpoint +ALTER TABLE "system_settings" ADD COLUMN "sticky_sla_ms" integer DEFAULT 20000 NOT NULL;--> statement-breakpoint +ALTER TABLE "system_settings" ADD COLUMN "racing_total_timeout_ms" integer DEFAULT 60000 NOT NULL;--> statement-breakpoint +ALTER TABLE "system_settings" ADD COLUMN "sticky_timeout_cooldown_ms" integer DEFAULT 300000 NOT NULL; \ No newline at end of file diff --git a/drizzle/meta/0110_snapshot.json b/drizzle/meta/0110_snapshot.json new file mode 100644 index 000000000..66d670d57 --- /dev/null +++ b/drizzle/meta/0110_snapshot.json @@ -0,0 +1,4691 @@ +{ + "id": "b6b7996d-5b70-4a31-a1c7-3a318b3398a0", + "prevId": "c054c34a-98a4-4ae1-b0e5-0b663380f123", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.audit_log": { + "name": "audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "action_category": { + "name": "action_category", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true + }, + "action_type": { + "name": "action_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "target_name": { + "name": "target_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "before_value": { + "name": "before_value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "after_value": { + "name": "after_value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "operator_user_id": { + "name": "operator_user_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operator_user_name": { + "name": "operator_user_name", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "operator_key_id": { + "name": "operator_key_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operator_key_name": { + "name": "operator_key_name", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "operator_ip": { + "name": "operator_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "success": { + "name": "success", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_audit_log_category_created_at": { + "name": "idx_audit_log_category_created_at", + "columns": [ + { + "expression": "action_category", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_operator_user_created_at": { + "name": "idx_audit_log_operator_user_created_at", + "columns": [ + { + "expression": "operator_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"operator_user_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_operator_ip_created_at": { + "name": "idx_audit_log_operator_ip_created_at", + "columns": [ + { + "expression": "operator_ip", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"operator_ip\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_target": { + "name": "idx_audit_log_target", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"target_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_created_at_id": { + "name": "idx_audit_log_created_at_id", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloud_pricing_catalog": { + "name": "cloud_pricing_catalog", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "default": "'USD'" + }, + "refreshed_at": { + "name": "refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "providers": { + "name": "providers", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "vendors": { + "name": "vendors", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "model_count": { + "name": "model_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "synced_at": { + "name": "synced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_rules": { + "name": "error_rules", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'regex'" + }, + "category": { + "name": "category", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "override_response": { + "name": "override_response", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "override_status_code": { + "name": "override_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_error_rules_enabled": { + "name": "idx_error_rules_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "unique_pattern": { + "name": "unique_pattern", + "columns": [ + { + "expression": "pattern", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_category": { + "name": "idx_category", + "columns": [ + { + "expression": "category", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_match_type": { + "name": "idx_match_type", + "columns": [ + { + "expression": "match_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.keys": { + "name": "keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "can_login_web_ui": { + "name": "can_login_web_ui", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_daily_usd": { + "name": "limit_daily_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "cost_reset_at": { + "name": "cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "provider_group": { + "name": "provider_group", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false, + "default": "'default'" + }, + "cache_ttl_preference": { + "name": "cache_ttl_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_keys_user_id": { + "name": "idx_keys_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_key": { + "name": "idx_keys_key", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_created_at": { + "name": "idx_keys_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_deleted_at": { + "name": "idx_keys_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.message_request": { + "name": "message_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "numeric(21, 15)", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "group_cost_multiplier": { + "name": "group_cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "cost_breakdown": { + "name": "cost_breakdown", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "request_sequence": { + "name": "request_sequence", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "provider_chain": { + "name": "provider_chain", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "api_type": { + "name": "api_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "endpoint": { + "name": "endpoint", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "original_model": { + "name": "original_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "actual_response_model": { + "name": "actual_response_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "ttfb_ms": { + "name": "ttfb_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cache_creation_input_tokens": { + "name": "cache_creation_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_read_input_tokens": { + "name": "cache_read_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_5m_input_tokens": { + "name": "cache_creation_5m_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_1h_input_tokens": { + "name": "cache_creation_1h_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_applied": { + "name": "cache_ttl_applied", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "context_1m_applied": { + "name": "context_1m_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "swap_cache_ttl_applied": { + "name": "swap_cache_ttl_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "special_settings": { + "name": "special_settings", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "hedge_losers": { + "name": "hedge_losers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_stack": { + "name": "error_stack", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_cause": { + "name": "error_cause", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "blocked_by": { + "name": "blocked_by", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "blocked_reason": { + "name": "blocked_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "client_ip": { + "name": "client_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "messages_count": { + "name": "messages_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_message_request_user_date_cost": { + "name": "idx_message_request_user_date_cost", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_created_at_cost_stats": { + "name": "idx_message_request_user_created_at_cost_stats", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_query": { + "name": "idx_message_request_user_query", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_created_at_active": { + "name": "idx_message_request_provider_created_at_active", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_created_at_finalized_active": { + "name": "idx_message_request_provider_created_at_finalized_active", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"status_code\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_id": { + "name": "idx_message_request_session_id", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_id_prefix": { + "name": "idx_message_request_session_id_prefix", + "columns": [ + { + "expression": "\"session_id\" varchar_pattern_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_seq": { + "name": "idx_message_request_session_seq", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_sequence", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_endpoint": { + "name": "idx_message_request_endpoint", + "columns": [ + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_blocked_by": { + "name": "idx_message_request_blocked_by", + "columns": [ + { + "expression": "blocked_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_id": { + "name": "idx_message_request_provider_id", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_id": { + "name": "idx_message_request_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key": { + "name": "idx_message_request_key", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_created_at_id": { + "name": "idx_message_request_key_created_at_id", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_model_active": { + "name": "idx_message_request_key_model_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"model\" IS NOT NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_endpoint_active": { + "name": "idx_message_request_key_endpoint_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"endpoint\" IS NOT NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_created_at_id_active": { + "name": "idx_message_request_created_at_id_active", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_model_active": { + "name": "idx_message_request_model_active", + "columns": [ + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"model\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_status_code_active": { + "name": "idx_message_request_status_code_active", + "columns": [ + { + "expression": "status_code", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"status_code\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_created_at": { + "name": "idx_message_request_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_deleted_at": { + "name": "idx_message_request_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_last_active": { + "name": "idx_message_request_key_last_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_cost_active": { + "name": "idx_message_request_key_cost_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_user_info": { + "name": "idx_message_request_session_user_info", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_client_ip_created_at": { + "name": "idx_message_request_client_ip_created_at", + "columns": [ + { + "expression": "client_ip", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"client_ip\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_prices": { + "name": "model_prices", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "model_name": { + "name": "model_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "price_data": { + "name": "price_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'cloud'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_model_prices_latest": { + "name": "idx_model_prices_latest", + "columns": [ + { + "expression": "model_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_model_name": { + "name": "idx_model_prices_model_name", + "columns": [ + { + "expression": "model_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_created_at": { + "name": "idx_model_prices_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_source": { + "name": "idx_model_prices_source", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_vendor": { + "name": "idx_model_prices_vendor", + "columns": [ + { + "expression": "((\"price_data\" ->> 'vendor'))", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_aliases": { + "name": "idx_model_prices_aliases", + "columns": [ + { + "expression": "((\"price_data\" -> 'aliases'))", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_settings": { + "name": "notification_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "use_legacy_mode": { + "name": "use_legacy_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "circuit_breaker_enabled": { + "name": "circuit_breaker_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "circuit_breaker_webhook": { + "name": "circuit_breaker_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "daily_leaderboard_enabled": { + "name": "daily_leaderboard_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "daily_leaderboard_webhook": { + "name": "daily_leaderboard_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "daily_leaderboard_time": { + "name": "daily_leaderboard_time", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false, + "default": "'09:00'" + }, + "daily_leaderboard_top_n": { + "name": "daily_leaderboard_top_n", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "cost_alert_enabled": { + "name": "cost_alert_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cost_alert_webhook": { + "name": "cost_alert_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "cost_alert_threshold": { + "name": "cost_alert_threshold", + "type": "numeric(5, 2)", + "primaryKey": false, + "notNull": false, + "default": "'0.80'" + }, + "cost_alert_check_interval": { + "name": "cost_alert_check_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 60 + }, + "cache_hit_rate_alert_enabled": { + "name": "cache_hit_rate_alert_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cache_hit_rate_alert_webhook": { + "name": "cache_hit_rate_alert_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "cache_hit_rate_alert_window_mode": { + "name": "cache_hit_rate_alert_window_mode", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false, + "default": "'auto'" + }, + "cache_hit_rate_alert_check_interval": { + "name": "cache_hit_rate_alert_check_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "cache_hit_rate_alert_historical_lookback_days": { + "name": "cache_hit_rate_alert_historical_lookback_days", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 7 + }, + "cache_hit_rate_alert_min_eligible_requests": { + "name": "cache_hit_rate_alert_min_eligible_requests", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 20 + }, + "cache_hit_rate_alert_min_eligible_tokens": { + "name": "cache_hit_rate_alert_min_eligible_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "cache_hit_rate_alert_abs_min": { + "name": "cache_hit_rate_alert_abs_min", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "cache_hit_rate_alert_drop_rel": { + "name": "cache_hit_rate_alert_drop_rel", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.3'" + }, + "cache_hit_rate_alert_drop_abs": { + "name": "cache_hit_rate_alert_drop_abs", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.1'" + }, + "cache_hit_rate_alert_cooldown_minutes": { + "name": "cache_hit_rate_alert_cooldown_minutes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30 + }, + "cache_hit_rate_alert_top_n": { + "name": "cache_hit_rate_alert_top_n", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_target_bindings": { + "name": "notification_target_bindings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "notification_type": { + "name": "notification_type", + "type": "notification_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "schedule_cron": { + "name": "schedule_cron", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "schedule_timezone": { + "name": "schedule_timezone", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "template_override": { + "name": "template_override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "unique_notification_target_binding": { + "name": "unique_notification_target_binding", + "columns": [ + { + "expression": "notification_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_notification_bindings_type": { + "name": "idx_notification_bindings_type", + "columns": [ + { + "expression": "notification_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_notification_bindings_target": { + "name": "idx_notification_bindings_target", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "notification_target_bindings_target_id_webhook_targets_id_fk": { + "name": "notification_target_bindings_target_id_webhook_targets_id_fk", + "tableFrom": "notification_target_bindings", + "tableTo": "webhook_targets", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_endpoint_probe_logs": { + "name": "provider_endpoint_probe_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "endpoint_id": { + "name": "endpoint_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'scheduled'" + }, + "ok": { + "name": "ok", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error_type": { + "name": "error_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_provider_endpoint_probe_logs_endpoint_created_at": { + "name": "idx_provider_endpoint_probe_logs_endpoint_created_at", + "columns": [ + { + "expression": "endpoint_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoint_probe_logs_created_at": { + "name": "idx_provider_endpoint_probe_logs_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_endpoint_probe_logs_endpoint_id_provider_endpoints_id_fk": { + "name": "provider_endpoint_probe_logs_endpoint_id_provider_endpoints_id_fk", + "tableFrom": "provider_endpoint_probe_logs", + "tableTo": "provider_endpoints", + "columnsFrom": [ + "endpoint_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_endpoints": { + "name": "provider_endpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "vendor_id": { + "name": "vendor_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'claude'" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_probed_at": { + "name": "last_probed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_probe_ok": { + "name": "last_probe_ok", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "last_probe_status_code": { + "name": "last_probe_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_probe_latency_ms": { + "name": "last_probe_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_probe_error_type": { + "name": "last_probe_error_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "last_probe_error_message": { + "name": "last_probe_error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "uniq_provider_endpoints_vendor_type_url": { + "name": "uniq_provider_endpoints_vendor_type_url", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_vendor_type": { + "name": "idx_provider_endpoints_vendor_type", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_enabled": { + "name": "idx_provider_endpoints_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_pick_enabled": { + "name": "idx_provider_endpoints_pick_enabled", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_created_at": { + "name": "idx_provider_endpoints_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_deleted_at": { + "name": "idx_provider_endpoints_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_endpoints_vendor_id_provider_vendors_id_fk": { + "name": "provider_endpoints_vendor_id_provider_vendors_id_fk", + "tableFrom": "provider_endpoints", + "tableTo": "provider_vendors", + "columnsFrom": [ + "vendor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_groups": { + "name": "provider_groups", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(200)", + "primaryKey": false, + "notNull": true + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": true, + "default": "'1.0'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_groups_name_unique": { + "name": "provider_groups_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_vendors": { + "name": "provider_vendors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "website_domain": { + "name": "website_domain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false + }, + "website_url": { + "name": "website_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "favicon_url": { + "name": "favicon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "uniq_provider_vendors_website_domain": { + "name": "uniq_provider_vendors_website_domain", + "columns": [ + { + "expression": "website_domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_vendors_created_at": { + "name": "idx_provider_vendors_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.providers": { + "name": "providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider_vendor_id": { + "name": "provider_vendor_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "weight": { + "name": "weight", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "group_priorities": { + "name": "group_priorities", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false, + "default": "'1.0'" + }, + "group_tag": { + "name": "group_tag", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "provider_type": { + "name": "provider_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'claude'" + }, + "preserve_client_ip": { + "name": "preserve_client_ip", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "disable_session_reuse": { + "name": "disable_session_reuse", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "model_redirects": { + "name": "model_redirects", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "allowed_models": { + "name": "allowed_models", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "allowed_clients": { + "name": "allowed_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "blocked_clients": { + "name": "blocked_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "active_time_start": { + "name": "active_time_start", + "type": "varchar(5)", + "primaryKey": false, + "notNull": false + }, + "active_time_end": { + "name": "active_time_end", + "type": "varchar(5)", + "primaryKey": false, + "notNull": false + }, + "codex_instructions_strategy": { + "name": "codex_instructions_strategy", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false, + "default": "'auto'" + }, + "mcp_passthrough_type": { + "name": "mcp_passthrough_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "mcp_passthrough_url": { + "name": "mcp_passthrough_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_daily_usd": { + "name": "limit_daily_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "total_cost_reset_at": { + "name": "total_cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "max_retry_attempts": { + "name": "max_retry_attempts", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "circuit_breaker_failure_threshold": { + "name": "circuit_breaker_failure_threshold", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "circuit_breaker_open_duration": { + "name": "circuit_breaker_open_duration", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1800000 + }, + "circuit_breaker_half_open_success_threshold": { + "name": "circuit_breaker_half_open_success_threshold", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 2 + }, + "proxy_url": { + "name": "proxy_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "proxy_fallback_to_direct": { + "name": "proxy_fallback_to_direct", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "custom_headers": { + "name": "custom_headers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "first_byte_timeout_streaming_ms": { + "name": "first_byte_timeout_streaming_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "streaming_idle_timeout_ms": { + "name": "streaming_idle_timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "request_timeout_non_streaming_ms": { + "name": "request_timeout_non_streaming_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "website_url": { + "name": "website_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "favicon_url": { + "name": "favicon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_preference": { + "name": "cache_ttl_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "swap_cache_ttl_billing": { + "name": "swap_cache_ttl_billing", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "context_1m_preference": { + "name": "context_1m_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_reasoning_effort_preference": { + "name": "codex_reasoning_effort_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_reasoning_summary_preference": { + "name": "codex_reasoning_summary_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_text_verbosity_preference": { + "name": "codex_text_verbosity_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_parallel_tool_calls_preference": { + "name": "codex_parallel_tool_calls_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_image_generation_preference": { + "name": "codex_image_generation_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_service_tier_preference": { + "name": "codex_service_tier_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_max_tokens_preference": { + "name": "anthropic_max_tokens_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_thinking_budget_preference": { + "name": "anthropic_thinking_budget_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_adaptive_thinking": { + "name": "anthropic_adaptive_thinking", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "gemini_google_search_preference": { + "name": "gemini_google_search_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "tpm": { + "name": "tpm", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "rpm": { + "name": "rpm", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "rpd": { + "name": "rpd", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "cc": { + "name": "cc", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_providers_enabled_priority": { + "name": "idx_providers_enabled_priority", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "weight", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_group": { + "name": "idx_providers_group", + "columns": [ + { + "expression": "group_tag", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_vendor_type_url_active": { + "name": "idx_providers_vendor_type_url_active", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_created_at": { + "name": "idx_providers_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_deleted_at": { + "name": "idx_providers_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_vendor_type": { + "name": "idx_providers_vendor_type", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_enabled_vendor_type": { + "name": "idx_providers_enabled_vendor_type", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL AND \"providers\".\"is_enabled\" = true AND \"providers\".\"provider_vendor_id\" IS NOT NULL AND \"providers\".\"provider_vendor_id\" > 0", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "providers_provider_vendor_id_provider_vendors_id_fk": { + "name": "providers_provider_vendor_id_provider_vendors_id_fk", + "tableFrom": "providers", + "tableTo": "provider_vendors", + "columnsFrom": [ + "provider_vendor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.request_filters": { + "name": "request_filters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "varchar(30)", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "target": { + "name": "target", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "replacement": { + "name": "replacement", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "binding_type": { + "name": "binding_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'global'" + }, + "provider_ids": { + "name": "provider_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "group_tags": { + "name": "group_tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "rule_mode": { + "name": "rule_mode", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'simple'" + }, + "execution_phase": { + "name": "execution_phase", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'guard'" + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_request_filters_enabled": { + "name": "idx_request_filters_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_scope": { + "name": "idx_request_filters_scope", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_action": { + "name": "idx_request_filters_action", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_binding": { + "name": "idx_request_filters_binding", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "binding_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_phase": { + "name": "idx_request_filters_phase", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_phase", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sensitive_words": { + "name": "sensitive_words", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "word": { + "name": "word", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'contains'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_sensitive_words_enabled": { + "name": "idx_sensitive_words_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "match_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_sensitive_words_created_at": { + "name": "idx_sensitive_words_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "site_title": { + "name": "site_title", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true, + "default": "'Claude Code Hub'" + }, + "allow_global_usage_view": { + "name": "allow_global_usage_view", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "currency_display": { + "name": "currency_display", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "default": "'USD'" + }, + "billing_model_source": { + "name": "billing_model_source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'original'" + }, + "codex_priority_billing_source": { + "name": "codex_priority_billing_source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'requested'" + }, + "bill_non_successful_requests": { + "name": "bill_non_successful_requests", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "bill_hedge_losers": { + "name": "bill_hedge_losers", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "discovery_enabled": { + "name": "discovery_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "discovery_concurrency": { + "name": "discovery_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "max_discovery_rounds": { + "name": "max_discovery_rounds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "discovery_sla_ms": { + "name": "discovery_sla_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10000 + }, + "sticky_sla_ms": { + "name": "sticky_sla_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 20000 + }, + "racing_total_timeout_ms": { + "name": "racing_total_timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 60000 + }, + "sticky_timeout_cooldown_ms": { + "name": "sticky_timeout_cooldown_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 300000 + }, + "timezone": { + "name": "timezone", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "enable_auto_cleanup": { + "name": "enable_auto_cleanup", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "cleanup_retention_days": { + "name": "cleanup_retention_days", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30 + }, + "cleanup_schedule": { + "name": "cleanup_schedule", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false, + "default": "'0 2 * * *'" + }, + "cleanup_batch_size": { + "name": "cleanup_batch_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10000 + }, + "enable_client_version_check": { + "name": "enable_client_version_check", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "verbose_provider_error": { + "name": "verbose_provider_error", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "pass_through_upstream_error_message": { + "name": "pass_through_upstream_error_message", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_http2": { + "name": "enable_http2", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "enable_openai_responses_websocket": { + "name": "enable_openai_responses_websocket", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_high_concurrency_mode": { + "name": "enable_high_concurrency_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "intercept_anthropic_warmup_requests": { + "name": "intercept_anthropic_warmup_requests", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "enable_thinking_signature_rectifier": { + "name": "enable_thinking_signature_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_thinking_budget_rectifier": { + "name": "enable_thinking_budget_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_thinking_effort_conflict_rectifier": { + "name": "enable_thinking_effort_conflict_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_gemini_function_id_rectifier": { + "name": "enable_gemini_function_id_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_billing_header_rectifier": { + "name": "enable_billing_header_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_response_input_rectifier": { + "name": "enable_response_input_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "allow_non_conversation_endpoint_provider_fallback": { + "name": "allow_non_conversation_endpoint_provider_fallback", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "fake_streaming_whitelist": { + "name": "fake_streaming_whitelist", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "enable_codex_session_id_completion": { + "name": "enable_codex_session_id_completion", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_claude_metadata_user_id_injection": { + "name": "enable_claude_metadata_user_id_injection", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_response_fixer": { + "name": "enable_response_fixer", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "response_fixer_config": { + "name": "response_fixer_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{\"fixTruncatedJson\":true,\"fixSseFormat\":true,\"fixEncoding\":true,\"maxJsonDepth\":200,\"maxFixSize\":1048576}'::jsonb" + }, + "quota_db_refresh_interval_seconds": { + "name": "quota_db_refresh_interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "quota_lease_percent_5h": { + "name": "quota_lease_percent_5h", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_daily": { + "name": "quota_lease_percent_daily", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_weekly": { + "name": "quota_lease_percent_weekly", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_monthly": { + "name": "quota_lease_percent_monthly", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_cap_usd": { + "name": "quota_lease_cap_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "ip_extraction_config": { + "name": "ip_extraction_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ip_geo_lookup_enabled": { + "name": "ip_geo_lookup_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "public_status_window_hours": { + "name": "public_status_window_hours", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 24 + }, + "public_status_aggregation_interval_minutes": { + "name": "public_status_aggregation_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_ledger": { + "name": "usage_ledger", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "final_provider_id": { + "name": "final_provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "original_model": { + "name": "original_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "actual_response_model": { + "name": "actual_response_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "endpoint": { + "name": "endpoint", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "api_type": { + "name": "api_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "is_success": { + "name": "is_success", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "success_rate_outcome": { + "name": "success_rate_outcome", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false + }, + "blocked_by": { + "name": "blocked_by", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "numeric(21, 15)", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "group_cost_multiplier": { + "name": "group_cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_input_tokens": { + "name": "cache_creation_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_read_input_tokens": { + "name": "cache_read_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_5m_input_tokens": { + "name": "cache_creation_5m_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_1h_input_tokens": { + "name": "cache_creation_1h_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_applied": { + "name": "cache_ttl_applied", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "context_1m_applied": { + "name": "context_1m_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "swap_cache_ttl_applied": { + "name": "swap_cache_ttl_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "ttfb_ms": { + "name": "ttfb_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "client_ip": { + "name": "client_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_usage_ledger_request_id": { + "name": "idx_usage_ledger_request_id", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_user_created_at": { + "name": "idx_usage_ledger_user_created_at", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_created_at": { + "name": "idx_usage_ledger_key_created_at", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_provider_created_at": { + "name": "idx_usage_ledger_provider_created_at", + "columns": [ + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_created_at_minute": { + "name": "idx_usage_ledger_created_at_minute", + "columns": [ + { + "expression": "date_trunc('minute', \"created_at\" AT TIME ZONE 'UTC')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_created_at_desc_id": { + "name": "idx_usage_ledger_created_at_desc_id", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_session_id": { + "name": "idx_usage_ledger_session_id", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"session_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_model": { + "name": "idx_usage_ledger_model", + "columns": [ + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"model\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_cost": { + "name": "idx_usage_ledger_key_cost", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_user_cost_cover": { + "name": "idx_usage_ledger_user_cost_cover", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_provider_cost_cover": { + "name": "idx_usage_ledger_provider_cost_cover", + "columns": [ + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_created_at_desc_cover": { + "name": "idx_usage_ledger_key_created_at_desc_cover", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"created_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "varchar", + "primaryKey": false, + "notNull": false, + "default": "'user'" + }, + "rpm_limit": { + "name": "rpm_limit", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "daily_limit_usd": { + "name": "daily_limit_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "provider_group": { + "name": "provider_group", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false, + "default": "'default'" + }, + "tags": { + "name": "tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "cost_reset_at": { + "name": "cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_5h_cost_reset_at": { + "name": "limit_5h_cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "allowed_clients": { + "name": "allowed_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "allowed_models": { + "name": "allowed_models", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "blocked_clients": { + "name": "blocked_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_users_active_role_sort": { + "name": "idx_users_active_role_sort", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "role", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_enabled_expires_at": { + "name": "idx_users_enabled_expires_at", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_tags_gin": { + "name": "idx_users_tags_gin", + "columns": [ + { + "expression": "tags", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "gin", + "with": {} + }, + "idx_users_created_at": { + "name": "idx_users_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_deleted_at": { + "name": "idx_users_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_targets": { + "name": "webhook_targets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "webhook_provider_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "webhook_url": { + "name": "webhook_url", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "telegram_bot_token": { + "name": "telegram_bot_token", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "telegram_chat_id": { + "name": "telegram_chat_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "dingtalk_secret": { + "name": "dingtalk_secret", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "custom_template": { + "name": "custom_template", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "custom_headers": { + "name": "custom_headers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "proxy_url": { + "name": "proxy_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "proxy_fallback_to_direct": { + "name": "proxy_fallback_to_direct", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_test_at": { + "name": "last_test_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_test_result": { + "name": "last_test_result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.daily_reset_mode": { + "name": "daily_reset_mode", + "schema": "public", + "values": [ + "fixed", + "rolling" + ] + }, + "public.notification_type": { + "name": "notification_type", + "schema": "public", + "values": [ + "circuit_breaker", + "daily_leaderboard", + "cost_alert", + "cache_hit_rate_alert" + ] + }, + "public.webhook_provider_type": { + "name": "webhook_provider_type", + "schema": "public", + "values": [ + "wechat", + "feishu", + "dingtalk", + "telegram", + "custom" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index a838a044c..b4b89464d 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -764,6 +764,13 @@ "when": 1783320834802, "tag": "0108_rich_onslaught", "breakpoints": true + }, + { + "idx": 110, + "version": "7", + "when": 1784571513591, + "tag": "0110_daffy_rawhide_kid", + "breakpoints": true } ] -} \ No newline at end of file +} diff --git a/messages/en/settings/config.json b/messages/en/settings/config.json index 0440d2c83..ad2fe21e8 100644 --- a/messages/en/settings/config.json +++ b/messages/en/settings/config.json @@ -118,6 +118,16 @@ "billHedgeLosers": "Bill Provider-Racing Losers by Token Usage", "billHedgeLosersDesc": "When provider racing (streaming hedge) is on, losing providers are kept connected in the background, drained for their token usage, and billed - their cost is added into this request's total. Default on.", "billHedgeLosersTooltip": "Upstreams usually bill a request even after we cancel it. Keeping racing losers alive lets us reclaim their token counts so CCH's cost matches what every upstream actually charged. Each loser's cost is accumulated asynchronously into the request total.", + "discoveryEnabled": "Enable bounded provider Discovery", + "discoveryEnabledDesc": "When enabled, cold-start streaming requests probe multiple providers within a bounded window and keep at most one fallback. It is disabled by default.", + "discoveryConcurrency": "Discovery initial concurrency", + "maxDiscoveryRounds": "Discovery maximum rounds", + "discoverySlaMs": "Discovery SLA (milliseconds)", + "stickySlaMs": "Sticky SLA (milliseconds)", + "racingTotalTimeoutMs": "Discovery total timeout (milliseconds)", + "stickyTimeoutCooldownMs": "Sticky timeout cooldown (milliseconds)", + "discoveryWindowDesc": "The total timeout must be at least Sticky SLA + maximum rounds × Discovery SLA. Discovery losers are cancelled and are not drained or billed by the legacy Hedge path.", + "discoveryWindowInvalid": "Discovery total timeout is shorter than the configured Sticky and Discovery windows.", "verboseProviderError": "Verbose Provider Error", "verboseProviderErrorDesc": "When enabled, CCH may return detailed diagnostic information for some upstream failure types in `error.details` (for example provider availability diagnostics or sanitized upstream snippets).", "verboseProviderErrorTooltip": "May expose provider names, internal routing clues, upstream failure reasons, and other diagnostic details. Enable only if clients are allowed to see low-level troubleshooting context.", diff --git a/messages/ja/settings/config.json b/messages/ja/settings/config.json index 0fc81c84a..b7ca0000b 100644 --- a/messages/ja/settings/config.json +++ b/messages/ja/settings/config.json @@ -120,6 +120,16 @@ "billHedgeLosers": "プロバイダー競争（hedge）の敗者を Token 使用量で課金", "billHedgeLosersDesc": "プロバイダー競争（ストリーミング hedge）が有効な場合、競争に敗れたプロバイダーを即座に切断せず、バックグラウンドで接続を維持して token 使用量を取得し課金します。その費用はこのリクエストの合計に加算されます。既定はオン。", "billHedgeLosersTooltip": "上流はこちらが能動的にキャンセルしたリクエストも通常は課金します。競争の敗者を生かしておくことで token 数を回収し、CCH の課金を各上流の実際の課金と一致させます。各敗者の費用は非同期にリクエストの費用へ加算されます。", + "discoveryEnabled": "制限付き Provider Discovery を有効化", + "discoveryEnabledDesc": "有効にすると、コールドスタートのストリーミングリクエストで複数 Provider を制限時間内に探索し、フォールバックを最大 1 つ保持します。既定はオフです。", + "discoveryConcurrency": "Discovery 初期並列数", + "maxDiscoveryRounds": "Discovery 最大ラウンド数", + "discoverySlaMs": "Discovery SLA（ミリ秒）", + "stickySlaMs": "Sticky SLA（ミリ秒）", + "racingTotalTimeoutMs": "Discovery 合計タイムアウト（ミリ秒）", + "stickyTimeoutCooldownMs": "Sticky タイムアウト後のクールダウン（ミリ秒）", + "discoveryWindowDesc": "合計タイムアウトは Sticky SLA + 最大ラウンド数 × Discovery SLA 以上にしてください。", + "discoveryWindowInvalid": "Discovery 合計タイムアウトが設定された Sticky/Discovery ウィンドウより短くなっています。", "verboseProviderError": "詳細なプロバイダーエラー", "verboseProviderErrorDesc": "有効にすると、一部の上流障害タイプで `error.details` により詳細な診断情報（プロバイダー可用性の診断やサニタイズ済み上流断片など）を含める場合があります。", "verboseProviderErrorTooltip": "この設定を有効にすると、プロバイダー名、内部ルーティングの手掛かり、上流障害の理由などの診断情報が露出する可能性があります。クライアントに低レベルのトラブルシュート文脈を見せてもよい場合にのみ有効化してください。", diff --git a/messages/ru/settings/config.json b/messages/ru/settings/config.json index 05e3dc217..108a2e4f8 100644 --- a/messages/ru/settings/config.json +++ b/messages/ru/settings/config.json @@ -120,6 +120,16 @@ "billHedgeLosers": "Тарифицировать проигравших в гонке провайдеров по токенам", "billHedgeLosersDesc": "Когда включена гонка провайдеров (streaming hedge), проигравшие провайдеры не отключаются сразу, а остаются на связи в фоне, дочитываются для получения использования токенов и тарифицируются - их стоимость добавляется в общую стоимость этого запроса. По умолчанию включено.", "billHedgeLosersTooltip": "Апстримы обычно тарифицируют запрос, даже если мы его отменили. Сохраняя проигравших в гонке, мы получаем их счётчики токенов, чтобы расходы CCH совпадали с тем, что фактически списал каждый апстрим. Стоимость каждого проигравшего асинхронно добавляется к стоимости запроса.", + "discoveryEnabled": "Включить ограниченное обнаружение провайдеров", + "discoveryEnabledDesc": "При включении потоковые запросы холодного старта проверяют несколько провайдеров в ограниченном окне и сохраняют не более одного резервного. По умолчанию выключено.", + "discoveryConcurrency": "Начальная параллельность Discovery", + "maxDiscoveryRounds": "Максимальное число раундов Discovery", + "discoverySlaMs": "SLA Discovery (миллисекунды)", + "stickySlaMs": "SLA Sticky (миллисекунды)", + "racingTotalTimeoutMs": "Общий тайм-аут Discovery (миллисекунды)", + "stickyTimeoutCooldownMs": "Пауза после тайм-аута Sticky (миллисекунды)", + "discoveryWindowDesc": "Общий тайм-аут должен быть не меньше SLA Sticky + максимальное число раундов × SLA Discovery.", + "discoveryWindowInvalid": "Общий тайм-аут Discovery меньше настроенного окна Sticky и Discovery.", "verboseProviderError": "Подробные ошибки провайдеров", "verboseProviderErrorDesc": "При включении CCH может добавлять более подробную диагностику некоторых типов сбоев апстрима в `error.details` (например, диагностику доступности провайдеров или очищенные фрагменты ответа апстрима).", "verboseProviderErrorTooltip": "Может раскрывать названия провайдеров, внутренние подсказки маршрутизации, причины сбоев апстрима и другие диагностические детали. Включайте только если клиентам допустимо видеть низкоуровневый контекст отладки.", diff --git a/messages/zh-CN/settings/config.json b/messages/zh-CN/settings/config.json index 19ea20509..b3249e8dd 100644 --- a/messages/zh-CN/settings/config.json +++ b/messages/zh-CN/settings/config.json @@ -47,6 +47,16 @@ "billHedgeLosers": "对供应商竞速输家计费", "billHedgeLosersDesc": "开启供应商竞速后，竞速落败的供应商不再被直接掐断，而是在后台保持连接、拿回其 token 用量并计费，其费用会累加进本条请求的总花费。默认开启。", "billHedgeLosersTooltip": "上游通常即使请求被我们主动取消也照样计费。保活竞速输家可以拿回它们的 token 计数，使 CCH 的扣费与每个上游实际扣费保持一致。每个输家的费用都会异步累加到该请求的花费中。", + "discoveryEnabled": "启用有界供应商 Discovery", + "discoveryEnabledDesc": "启用后，冷启动流式请求会在限定窗口内探测多个供应商，并且最多保留一个保底请求。默认关闭。", + "discoveryConcurrency": "Discovery 首轮并发数", + "maxDiscoveryRounds": "Discovery 最大轮数", + "discoverySlaMs": "Discovery SLA（毫秒）", + "stickySlaMs": "Sticky SLA（毫秒）", + "racingTotalTimeoutMs": "Discovery 总超时（毫秒）", + "stickyTimeoutCooldownMs": "Sticky 超时冷却（毫秒）", + "discoveryWindowDesc": "总超时必须不小于 Sticky SLA + 最大轮数 × Discovery SLA。Discovery 输家会取消，不走旧 Hedge 的 drain 或输家计费。", + "discoveryWindowInvalid": "Discovery 总超时短于已配置的 Sticky 与 Discovery 窗口。", "verboseProviderError": "详细供应商错误信息", "verboseProviderErrorDesc": "开启后，CCH 会在某些上游失败类型下于 `error.details` 返回更详细的诊断信息（例如供应商可用性诊断或脱敏后的上游片段）。", "verboseProviderErrorTooltip": "该选项可能暴露供应商名称、内部路由线索、上游失败原因等诊断信息。仅建议在客户端可以查看底层排障上下文时开启。", diff --git a/messages/zh-TW/settings/config.json b/messages/zh-TW/settings/config.json index 590ff7a77..e510dd0c1 100644 --- a/messages/zh-TW/settings/config.json +++ b/messages/zh-TW/settings/config.json @@ -120,6 +120,16 @@ "billHedgeLosers": "對供應商競速輸家計費", "billHedgeLosersDesc": "開啟供應商競速後，競速落敗的供應商不再被直接掐斷，而是在後台保持連線、取回其 token 用量並計費，其費用會累加進本條請求的總花費。預設開啟。", "billHedgeLosersTooltip": "上游通常即使請求被我們主動取消也照常計費。保活競速輸家可以取回它們的 token 計數，使 CCH 的扣費與每個上游實際扣費保持一致。每個輸家的費用都會非同步累加到該請求的花費中。", + "discoveryEnabled": "啟用有界供應商 Discovery", + "discoveryEnabledDesc": "啟用後，冷啟動串流請求會在限定視窗內探測多個供應商，並且最多保留一個保底請求。預設關閉。", + "discoveryConcurrency": "Discovery 首輪並發數", + "maxDiscoveryRounds": "Discovery 最大輪數", + "discoverySlaMs": "探索 SLA（毫秒）", + "stickySlaMs": "Sticky 黏性 SLA（毫秒）", + "racingTotalTimeoutMs": "Discovery 總逾時（毫秒）", + "stickyTimeoutCooldownMs": "Sticky 逾時冷卻（毫秒）", + "discoveryWindowDesc": "總逾時必須不小於 Sticky SLA + 最大輪數 × Discovery SLA。Discovery 輸家會取消，不走舊 Hedge 的 drain 或輸家計費。", + "discoveryWindowInvalid": "Discovery 總逾時短於已設定的 Sticky 與 Discovery 視窗。", "verboseProviderError": "詳細供應商錯誤資訊", "verboseProviderErrorDesc": "開啟後，CCH 會在某些上游失敗類型下於 `error.details` 返回較詳細的診斷資訊（例如供應商可用性診斷或脫敏後的上游片段）。", "verboseProviderErrorTooltip": "此選項可能暴露供應商名稱、內部路由線索、上游失敗原因等診斷資訊。僅建議在客戶端可以查看底層排障上下文時開啟。", diff --git a/src/actions/system-config.ts b/src/actions/system-config.ts index 5c42d12d0..39464d90b 100644 --- a/src/actions/system-config.ts +++ b/src/actions/system-config.ts @@ -64,6 +64,13 @@ export async function saveSystemSettings(formData: { codexPriorityBillingSource?: CodexPriorityBillingSource; billNonSuccessfulRequests?: boolean; billHedgeLosers?: boolean; + discoveryEnabled?: boolean; + discoveryConcurrency?: number; + maxDiscoveryRounds?: number; + discoverySlaMs?: number; + stickySlaMs?: number; + racingTotalTimeoutMs?: number; + stickyTimeoutCooldownMs?: number; timezone?: string | null; enableAutoCleanup?: boolean; cleanupRetentionDays?: number; @@ -110,6 +117,22 @@ export async function saveSystemSettings(formData: { before = await getSystemSettings(); const validated = UpdateSystemSettingsSchema.parse(formData); + const effectiveDiscoveryWindow = { + discoverySlaMs: validated.discoverySlaMs ?? before.discoverySlaMs, + stickySlaMs: validated.stickySlaMs ?? before.stickySlaMs, + maxDiscoveryRounds: validated.maxDiscoveryRounds ?? before.maxDiscoveryRounds, + racingTotalTimeoutMs: validated.racingTotalTimeoutMs ?? before.racingTotalTimeoutMs, + }; + if ( + effectiveDiscoveryWindow.racingTotalTimeoutMs < + effectiveDiscoveryWindow.stickySlaMs + + effectiveDiscoveryWindow.maxDiscoveryRounds * effectiveDiscoveryWindow.discoverySlaMs + ) { + return { + ok: false, + error: "竞速总超时必须不小于 Sticky SLA + Discovery 轮数 × Discovery SLA", + }; + } const updated = await updateSystemSettings({ siteTitle: validated.siteTitle?.trim(), allowGlobalUsageView: validated.allowGlobalUsageView, @@ -118,6 +141,13 @@ export async function saveSystemSettings(formData: { codexPriorityBillingSource: validated.codexPriorityBillingSource, billNonSuccessfulRequests: validated.billNonSuccessfulRequests, billHedgeLosers: validated.billHedgeLosers, + discoveryEnabled: validated.discoveryEnabled, + discoveryConcurrency: validated.discoveryConcurrency, + maxDiscoveryRounds: validated.maxDiscoveryRounds, + discoverySlaMs: validated.discoverySlaMs, + stickySlaMs: validated.stickySlaMs, + racingTotalTimeoutMs: validated.racingTotalTimeoutMs, + stickyTimeoutCooldownMs: validated.stickyTimeoutCooldownMs, timezone: validated.timezone, enableAutoCleanup: validated.enableAutoCleanup, cleanupRetentionDays: validated.cleanupRetentionDays, diff --git a/src/app/[locale]/settings/config/_components/system-settings-form.tsx b/src/app/[locale]/settings/config/_components/system-settings-form.tsx index dabd8e344..26a4b7858 100644 --- a/src/app/[locale]/settings/config/_components/system-settings-form.tsx +++ b/src/app/[locale]/settings/config/_components/system-settings-form.tsx @@ -65,6 +65,13 @@ interface SystemSettingsFormProps { | "codexPriorityBillingSource" | "billNonSuccessfulRequests" | "billHedgeLosers" + | "discoveryEnabled" + | "discoveryConcurrency" + | "maxDiscoveryRounds" + | "discoverySlaMs" + | "stickySlaMs" + | "racingTotalTimeoutMs" + | "stickyTimeoutCooldownMs" | "timezone" | "verboseProviderError" | "passThroughUpstreamErrorMessage" @@ -130,6 +137,19 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) initialSettings.billNonSuccessfulRequests ); const [billHedgeLosers, setBillHedgeLosers] = useState(initialSettings.billHedgeLosers); + const [discoveryEnabled, setDiscoveryEnabled] = useState(initialSettings.discoveryEnabled); + const [discoveryConcurrency, setDiscoveryConcurrency] = useState( + initialSettings.discoveryConcurrency + ); + const [maxDiscoveryRounds, setMaxDiscoveryRounds] = useState(initialSettings.maxDiscoveryRounds); + const [discoverySlaMs, setDiscoverySlaMs] = useState(initialSettings.discoverySlaMs); + const [stickySlaMs, setStickySlaMs] = useState(initialSettings.stickySlaMs); + const [racingTotalTimeoutMs, setRacingTotalTimeoutMs] = useState( + initialSettings.racingTotalTimeoutMs + ); + const [stickyTimeoutCooldownMs, setStickyTimeoutCooldownMs] = useState( + initialSettings.stickyTimeoutCooldownMs + ); const [timezone, setTimezone] = useState(initialSettings.timezone); const [verboseProviderError, setVerboseProviderError] = useState( initialSettings.verboseProviderError @@ -230,6 +250,11 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) return; } + if (racingTotalTimeoutMs < stickySlaMs + maxDiscoveryRounds * discoverySlaMs) { + toast.error(t("discoveryWindowInvalid")); + return; + } + const quotaDbRefreshIntervalSecondsToSave = clampQuotaDbRefreshIntervalSeconds( quotaDbRefreshIntervalSecondsStr ); @@ -311,6 +336,13 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) codexPriorityBillingSource, billNonSuccessfulRequests, billHedgeLosers, + discoveryEnabled, + discoveryConcurrency, + maxDiscoveryRounds, + discoverySlaMs, + stickySlaMs, + racingTotalTimeoutMs, + stickyTimeoutCooldownMs, timezone, verboseProviderError, passThroughUpstreamErrorMessage, @@ -353,6 +385,13 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) setCodexPriorityBillingSource(result.data.codexPriorityBillingSource); setBillNonSuccessfulRequests(result.data.billNonSuccessfulRequests); setBillHedgeLosers(result.data.billHedgeLosers); + setDiscoveryEnabled(result.data.discoveryEnabled); + setDiscoveryConcurrency(result.data.discoveryConcurrency); + setMaxDiscoveryRounds(result.data.maxDiscoveryRounds); + setDiscoverySlaMs(result.data.discoverySlaMs); + setStickySlaMs(result.data.stickySlaMs); + setRacingTotalTimeoutMs(result.data.racingTotalTimeoutMs); + setStickyTimeoutCooldownMs(result.data.stickyTimeoutCooldownMs); setTimezone(result.data.timezone); setVerboseProviderError(result.data.verboseProviderError); setPassThroughUpstreamErrorMessage(result.data.passThroughUpstreamErrorMessage); @@ -636,6 +675,56 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) /> + {/* Bounded Streaming Discovery */} +
+
+
+
+ +
+
+

{t("discoveryEnabled")}

+

{t("discoveryEnabledDesc")}

+
+
+ +
+
+ {( + [ + ["discoveryConcurrency", discoveryConcurrency, setDiscoveryConcurrency, 1], + ["maxDiscoveryRounds", maxDiscoveryRounds, setMaxDiscoveryRounds, 1], + ["discoverySlaMs", discoverySlaMs, setDiscoverySlaMs, 1], + ["stickySlaMs", stickySlaMs, setStickySlaMs, 1], + ["racingTotalTimeoutMs", racingTotalTimeoutMs, setRacingTotalTimeoutMs, 1], + ["stickyTimeoutCooldownMs", stickyTimeoutCooldownMs, setStickyTimeoutCooldownMs, 1], + ] as const + ).map(([key, value, setter, min]) => ( +
+ + setter(Number(event.target.value))} + disabled={isPending || !discoveryEnabled} + className={inputClassName} + /> +
+ ))} +
+

{t("discoveryWindowDesc")}

+
+ {/* Verbose Provider Error */}
diff --git a/src/app/[locale]/settings/config/page.tsx b/src/app/[locale]/settings/config/page.tsx index b5eade7e6..d80019bd0 100644 --- a/src/app/[locale]/settings/config/page.tsx +++ b/src/app/[locale]/settings/config/page.tsx @@ -52,6 +52,13 @@ async function SettingsConfigContent({ locale }: { locale: string }) { codexPriorityBillingSource: settings.codexPriorityBillingSource, billNonSuccessfulRequests: settings.billNonSuccessfulRequests, billHedgeLosers: settings.billHedgeLosers, + discoveryEnabled: settings.discoveryEnabled, + discoveryConcurrency: settings.discoveryConcurrency, + maxDiscoveryRounds: settings.maxDiscoveryRounds, + discoverySlaMs: settings.discoverySlaMs, + stickySlaMs: settings.stickySlaMs, + racingTotalTimeoutMs: settings.racingTotalTimeoutMs, + stickyTimeoutCooldownMs: settings.stickyTimeoutCooldownMs, timezone: settings.timezone, verboseProviderError: settings.verboseProviderError, passThroughUpstreamErrorMessage: settings.passThroughUpstreamErrorMessage, diff --git a/src/app/api/admin/system-config/route.ts b/src/app/api/admin/system-config/route.ts index e9abff730..a5a925e73 100644 --- a/src/app/api/admin/system-config/route.ts +++ b/src/app/api/admin/system-config/route.ts @@ -56,9 +56,20 @@ export async function POST(req: Request) { try { const body = await req.json(); + const current = await getSystemSettings(); // 验证请求数据 const validated = UpdateSystemSettingsSchema.parse(body); + const discoverySlaMs = validated.discoverySlaMs ?? current.discoverySlaMs; + const stickySlaMs = validated.stickySlaMs ?? current.stickySlaMs; + const maxDiscoveryRounds = validated.maxDiscoveryRounds ?? current.maxDiscoveryRounds; + const racingTotalTimeoutMs = validated.racingTotalTimeoutMs ?? current.racingTotalTimeoutMs; + if (racingTotalTimeoutMs < stickySlaMs + maxDiscoveryRounds * discoverySlaMs) { + return Response.json( + { error: "竞速总超时必须不小于 Sticky SLA + Discovery 轮数 × Discovery SLA" }, + { status: 400 } + ); + } // 更新系统设置 const updated = await updateSystemSettings({ @@ -67,6 +78,13 @@ export async function POST(req: Request) { currencyDisplay: validated.currencyDisplay, billingModelSource: validated.billingModelSource, codexPriorityBillingSource: validated.codexPriorityBillingSource, + discoveryEnabled: validated.discoveryEnabled, + discoveryConcurrency: validated.discoveryConcurrency, + maxDiscoveryRounds: validated.maxDiscoveryRounds, + discoverySlaMs: validated.discoverySlaMs, + stickySlaMs: validated.stickySlaMs, + racingTotalTimeoutMs: validated.racingTotalTimeoutMs, + stickyTimeoutCooldownMs: validated.stickyTimeoutCooldownMs, timezone: validated.timezone, enableAutoCleanup: validated.enableAutoCleanup, cleanupRetentionDays: validated.cleanupRetentionDays, diff --git a/src/drizzle/schema.ts b/src/drizzle/schema.ts index afba323ba..7534e86d2 100644 --- a/src/drizzle/schema.ts +++ b/src/drizzle/schema.ts @@ -780,6 +780,15 @@ export const systemSettings = pgTable('system_settings', { // 关闭：竞速输家直接取消连接，不计费（旧行为） billHedgeLosers: boolean('bill_hedge_losers').notNull().default(true), + // Bounded streaming Discovery (disabled by default until explicitly enabled). + discoveryEnabled: boolean('discovery_enabled').notNull().default(false), + discoveryConcurrency: integer('discovery_concurrency').notNull().default(2), + maxDiscoveryRounds: integer('max_discovery_rounds').notNull().default(2), + discoverySlaMs: integer('discovery_sla_ms').notNull().default(10000), + stickySlaMs: integer('sticky_sla_ms').notNull().default(20000), + racingTotalTimeoutMs: integer('racing_total_timeout_ms').notNull().default(60000), + stickyTimeoutCooldownMs: integer('sticky_timeout_cooldown_ms').notNull().default(300000), + // 系统时区配置 (IANA timezone identifier) // 用于统一后端时间边界计算和前端日期/时间显示 // null 表示使用环境变量 TZ 或默认 UTC diff --git a/src/lib/api-client/v1/openapi-types.gen.ts b/src/lib/api-client/v1/openapi-types.gen.ts index 8c5d007ca..dd0ab0996 100644 --- a/src/lib/api-client/v1/openapi-types.gen.ts +++ b/src/lib/api-client/v1/openapi-types.gen.ts @@ -11885,6 +11885,20 @@ export interface operations { billNonSuccessfulRequests: boolean; /** @description Whether streaming-hedge (provider racing) losers are kept alive, drained, and billed (their cost accumulates into the request total). */ billHedgeLosers: boolean; + /** @description Whether bounded streaming Discovery is enabled. */ + discoveryEnabled: boolean; + /** @description Maximum number of normal Discovery attempts in the initial batch. */ + discoveryConcurrency: number; + /** @description Maximum number of Discovery rounds. */ + maxDiscoveryRounds: number; + /** @description 首字 Discovery SLA in milliseconds. */ + discoverySlaMs: number; + /** @description Sticky probe SLA in milliseconds. */ + stickySlaMs: number; + /** @description Total pre-winner Discovery deadline in milliseconds. */ + racingTotalTimeoutMs: number; + /** @description Sticky timeout cooldown in milliseconds. */ + stickyTimeoutCooldownMs: number; /** @description Configured system timezone, or null for default. */ timezone: string | null; /** @description Whether usage-log cleanup is enabled. */ @@ -12147,6 +12161,20 @@ export interface operations { billNonSuccessfulRequests?: boolean; /** @description Whether streaming-hedge (provider racing) losers are kept alive, drained, and billed (their cost accumulates into the request total). */ billHedgeLosers?: boolean; + /** @description Whether bounded streaming Discovery is enabled. */ + discoveryEnabled?: boolean; + /** @description Maximum number of normal Discovery attempts in the initial batch. */ + discoveryConcurrency?: number; + /** @description Maximum number of Discovery rounds. */ + maxDiscoveryRounds?: number; + /** @description 首字 Discovery SLA in milliseconds. */ + discoverySlaMs?: number; + /** @description Sticky probe SLA in milliseconds. */ + stickySlaMs?: number; + /** @description Total pre-winner Discovery deadline in milliseconds. */ + racingTotalTimeoutMs?: number; + /** @description Sticky timeout cooldown in milliseconds. */ + stickyTimeoutCooldownMs?: number; /** @description System timezone, or null to use default. */ timezone?: string | null; /** @description Whether usage-log cleanup is enabled. */ @@ -12282,6 +12310,20 @@ export interface operations { billNonSuccessfulRequests: boolean; /** @description Whether streaming-hedge (provider racing) losers are kept alive, drained, and billed (their cost accumulates into the request total). */ billHedgeLosers: boolean; + /** @description Whether bounded streaming Discovery is enabled. */ + discoveryEnabled: boolean; + /** @description Maximum number of normal Discovery attempts in the initial batch. */ + discoveryConcurrency: number; + /** @description Maximum number of Discovery rounds. */ + maxDiscoveryRounds: number; + /** @description 首字 Discovery SLA in milliseconds. */ + discoverySlaMs: number; + /** @description Sticky probe SLA in milliseconds. */ + stickySlaMs: number; + /** @description Total pre-winner Discovery deadline in milliseconds. */ + racingTotalTimeoutMs: number; + /** @description Sticky timeout cooldown in milliseconds. */ + stickyTimeoutCooldownMs: number; /** @description Configured system timezone, or null for default. */ timezone: string | null; /** @description Whether usage-log cleanup is enabled. */ diff --git a/src/lib/api/v1/schemas/system-config.ts b/src/lib/api/v1/schemas/system-config.ts index 204c2db66..e52e21b17 100644 --- a/src/lib/api/v1/schemas/system-config.ts +++ b/src/lib/api/v1/schemas/system-config.ts @@ -98,6 +98,25 @@ export const SystemSettingsSchema = z .describe( "Whether streaming-hedge (provider racing) losers are kept alive, drained, and billed (their cost accumulates into the request total)." ), + discoveryEnabled: z.boolean().describe("Whether bounded streaming Discovery is enabled."), + discoveryConcurrency: z + .number() + .int() + .positive() + .describe("Maximum number of normal Discovery attempts in the initial batch."), + maxDiscoveryRounds: z.number().int().positive().describe("Maximum number of Discovery rounds."), + discoverySlaMs: z.number().int().positive().describe("首字 Discovery SLA in milliseconds."), + stickySlaMs: z.number().int().positive().describe("Sticky probe SLA in milliseconds."), + racingTotalTimeoutMs: z + .number() + .int() + .positive() + .describe("Total pre-winner Discovery deadline in milliseconds."), + stickyTimeoutCooldownMs: z + .number() + .int() + .positive() + .describe("Sticky timeout cooldown in milliseconds."), timezone: TimeZoneSchema.nullable().describe( "Configured system timezone, or null for default." ), diff --git a/src/lib/config/system-settings-cache.ts b/src/lib/config/system-settings-cache.ts index 6649603f5..0dd733b75 100644 --- a/src/lib/config/system-settings-cache.ts +++ b/src/lib/config/system-settings-cache.ts @@ -54,6 +54,13 @@ const DEFAULT_SETTINGS: Pick< | "passThroughUpstreamErrorMessage" | "publicStatusWindowHours" | "publicStatusAggregationIntervalMinutes" + | "discoveryEnabled" + | "discoveryConcurrency" + | "maxDiscoveryRounds" + | "discoverySlaMs" + | "stickySlaMs" + | "racingTotalTimeoutMs" + | "stickyTimeoutCooldownMs" > = { enableHttp2: false, enableOpenaiResponsesWebsocket: true, @@ -84,6 +91,13 @@ const DEFAULT_SETTINGS: Pick< }, publicStatusWindowHours: 24, publicStatusAggregationIntervalMinutes: 5, + discoveryEnabled: false, + discoveryConcurrency: 2, + maxDiscoveryRounds: 2, + discoverySlaMs: 10_000, + stickySlaMs: 20_000, + racingTotalTimeoutMs: 60_000, + stickyTimeoutCooldownMs: 300_000, }; /** @@ -166,13 +180,13 @@ export async function getCachedSystemSettings(): Promise { publicStatusWindowHours: DEFAULT_SETTINGS.publicStatusWindowHours, publicStatusAggregationIntervalMinutes: DEFAULT_SETTINGS.publicStatusAggregationIntervalMinutes, - discoveryEnabled: false, - discoveryConcurrency: 2, - maxDiscoveryRounds: 2, - discoverySlaMs: 10_000, - stickySlaMs: 20_000, - racingTotalTimeoutMs: 60_000, - stickyTimeoutCooldownMs: 300_000, + discoveryEnabled: DEFAULT_SETTINGS.discoveryEnabled, + discoveryConcurrency: DEFAULT_SETTINGS.discoveryConcurrency, + maxDiscoveryRounds: DEFAULT_SETTINGS.maxDiscoveryRounds, + discoverySlaMs: DEFAULT_SETTINGS.discoverySlaMs, + stickySlaMs: DEFAULT_SETTINGS.stickySlaMs, + racingTotalTimeoutMs: DEFAULT_SETTINGS.racingTotalTimeoutMs, + stickyTimeoutCooldownMs: DEFAULT_SETTINGS.stickyTimeoutCooldownMs, quotaDbRefreshIntervalSeconds: 10, quotaLeasePercent5h: 0.05, quotaLeasePercentDaily: 0.05, diff --git a/src/lib/validation/schemas.ts b/src/lib/validation/schemas.ts index bd9d51c0b..f347fc7ba 100644 --- a/src/lib/validation/schemas.ts +++ b/src/lib/validation/schemas.ts @@ -946,205 +946,270 @@ export const UpdateProviderSchema = z * 系统设置更新数据验证schema * 注意：所有字段均为可选，支持部分更新 */ -export const UpdateSystemSettingsSchema = z.object({ - siteTitle: z.string().min(1, "站点标题不能为空").max(128, "站点标题不能超过128个字符").optional(), - allowGlobalUsageView: z.boolean().optional(), - currencyDisplay: z - .enum( - Object.keys(CURRENCY_CONFIG) as [ - keyof typeof CURRENCY_CONFIG, - ...Array, - ], - { message: "不支持的货币类型" } - ) - .optional(), - // 计费模型来源配置（可选） - billingModelSource: z - .enum(["original", "redirected"], { message: "不支持的计费模型来源" }) - .optional(), - codexPriorityBillingSource: z - .enum(["requested", "actual"], { message: "不支持的 Codex Priority 计费来源" }) - .optional(), - // 系统时区配置（可选） - // 必须是有效的 IANA 时区标识符（如 "Asia/Shanghai", "America/New_York"） - timezone: z - .string() - .refine((val) => isValidIANATimezone(val), { - message: "无效的时区标识符，请使用 IANA 时区格式（如 Asia/Shanghai）", - }) - .nullable() - .optional(), - // 日志清理配置（可选） - enableAutoCleanup: z.boolean().optional(), - cleanupRetentionDays: z.coerce - .number() - .int("保留天数必须是整数") - .min(1, "保留天数不能少于1天") - .max(365, "保留天数不能超过365天") - .optional(), - cleanupSchedule: z.string().min(1, "执行时间不能为空").optional(), - cleanupBatchSize: z.coerce - .number() - .int("批量大小必须是整数") - .min(1000, "批量大小不能少于1000") - .max(100000, "批量大小不能超过100000") - .optional(), - // 客户端版本检查配置（可选） - enableClientVersionCheck: z.boolean().optional(), - // 供应商不可用时是否返回详细错误信息（可选） - verboseProviderError: z.boolean().optional(), - // 标准代理错误响应是否透传安全脱敏后的上游错误 message（可选） - passThroughUpstreamErrorMessage: z.boolean().optional(), - // 启用 HTTP/2 连接供应商（可选） - enableHttp2: z.boolean().optional(), - // 非成功请求按 token 用量计费（可选；默认关闭） - billNonSuccessfulRequests: z.boolean().optional(), - // 供应商竞速输家计费（可选；默认开启） - billHedgeLosers: z.boolean().optional(), - // 启用 OpenAI Responses WebSocket 支持（可选，仅 Codex 类型供应商生效） - enableOpenaiResponsesWebsocket: z.boolean().optional(), - // 高并发模式（可选） - enableHighConcurrencyMode: z.boolean().optional(), - // 可选拦截 Anthropic Warmup 请求（可选） - interceptAnthropicWarmupRequests: z.boolean().optional(), - // thinking signature 整流器（可选） - enableThinkingSignatureRectifier: z.boolean().optional(), - // thinking budget 整流器（可选） - enableThinkingBudgetRectifier: z.boolean().optional(), - // thinking effort 冲突整流器（可选） - enableThinkingEffortConflictRectifier: z.boolean().optional(), - // Gemini function id 整流器（可选） - enableGeminiFunctionIdRectifier: z.boolean().optional(), - // billing header 整流器（可选） - enableBillingHeaderRectifier: z.boolean().optional(), - // Response API input 整流器（可选） - enableResponseInputRectifier: z.boolean().optional(), - // 非对话端点跨供应商 fallback（可选） - allowNonConversationEndpointProviderFallback: z.boolean().optional(), - // Fake 流式输出白名单（可选）。空数组表示显式禁用；缺省 → 使用默认四个图像生成模型。 - fakeStreamingWhitelist: z - .array( - z.object({ - model: z - .string() - .min(1, "model 不能为空") - .max(200, "model 不能超过 200 个字符") - .transform((value) => value.trim()) - .refine((value) => value.length > 0, { message: "model 不能为空" }), - groupTags: z - .array( - z - .string() - .min(1) - .transform((value) => value.trim()) - .refine((value) => value.length > 0, { message: "groupTag 不能为空" }) - ) - .default([]) - .transform((tags) => Array.from(new Set(tags))), +export const UpdateSystemSettingsSchema = z + .object({ + siteTitle: z + .string() + .min(1, "站点标题不能为空") + .max(128, "站点标题不能超过128个字符") + .optional(), + allowGlobalUsageView: z.boolean().optional(), + currencyDisplay: z + .enum( + Object.keys(CURRENCY_CONFIG) as [ + keyof typeof CURRENCY_CONFIG, + ...Array, + ], + { message: "不支持的货币类型" } + ) + .optional(), + // 计费模型来源配置（可选） + billingModelSource: z + .enum(["original", "redirected"], { message: "不支持的计费模型来源" }) + .optional(), + codexPriorityBillingSource: z + .enum(["requested", "actual"], { message: "不支持的 Codex Priority 计费来源" }) + .optional(), + // 系统时区配置（可选） + // 必须是有效的 IANA 时区标识符（如 "Asia/Shanghai", "America/New_York"） + timezone: z + .string() + .refine((val) => isValidIANATimezone(val), { + message: "无效的时区标识符，请使用 IANA 时区格式（如 Asia/Shanghai）", }) - ) - .superRefine((entries, ctx) => { - const seen = new Set(); - for (let index = 0; index < entries.length; index += 1) { - const model = entries[index].model; - if (seen.has(model)) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: `fakeStreamingWhitelist 模型重复: ${model}`, - path: [index, "model"], - }); + .nullable() + .optional(), + // 日志清理配置（可选） + enableAutoCleanup: z.boolean().optional(), + cleanupRetentionDays: z.coerce + .number() + .int("保留天数必须是整数") + .min(1, "保留天数不能少于1天") + .max(365, "保留天数不能超过365天") + .optional(), + cleanupSchedule: z.string().min(1, "执行时间不能为空").optional(), + cleanupBatchSize: z.coerce + .number() + .int("批量大小必须是整数") + .min(1000, "批量大小不能少于1000") + .max(100000, "批量大小不能超过100000") + .optional(), + // 客户端版本检查配置（可选） + enableClientVersionCheck: z.boolean().optional(), + // 供应商不可用时是否返回详细错误信息（可选） + verboseProviderError: z.boolean().optional(), + // 标准代理错误响应是否透传安全脱敏后的上游错误 message（可选） + passThroughUpstreamErrorMessage: z.boolean().optional(), + // 启用 HTTP/2 连接供应商（可选） + enableHttp2: z.boolean().optional(), + // 非成功请求按 token 用量计费（可选；默认关闭） + billNonSuccessfulRequests: z.boolean().optional(), + // 供应商竞速输家计费（可选；默认开启） + billHedgeLosers: z.boolean().optional(), + // Bounded streaming Discovery（默认关闭；启用前需满足总窗口约束） + discoveryEnabled: z.boolean().optional(), + discoveryConcurrency: z.coerce + .number() + .int("Discovery 并发数必须是整数") + .min(1, "Discovery 并发数必须大于 0") + .max(32, "Discovery 并发数不能超过 32") + .optional(), + maxDiscoveryRounds: z.coerce + .number() + .int("Discovery 轮数必须是整数") + .min(1, "Discovery 轮数必须大于 0") + .max(32, "Discovery 轮数不能超过 32") + .optional(), + discoverySlaMs: z.coerce + .number() + .int("Discovery SLA 必须是整数毫秒") + .min(1, "Discovery SLA 必须大于 0") + .max(300_000, "Discovery SLA 不能超过 300000 毫秒") + .optional(), + stickySlaMs: z.coerce + .number() + .int("Sticky SLA 必须是整数毫秒") + .min(1, "Sticky SLA 必须大于 0") + .max(600_000, "Sticky SLA 不能超过 600000 毫秒") + .optional(), + racingTotalTimeoutMs: z.coerce + .number() + .int("竞速总超时必须是整数毫秒") + .min(1, "竞速总超时必须大于 0") + .max(3_600_000, "竞速总超时不能超过 3600000 毫秒") + .optional(), + stickyTimeoutCooldownMs: z.coerce + .number() + .int("Sticky 冷却时间必须是整数毫秒") + .min(1, "Sticky 冷却时间必须大于 0") + .max(86_400_000, "Sticky 冷却时间不能超过 86400000 毫秒") + .optional(), + // 启用 OpenAI Responses WebSocket 支持（可选，仅 Codex 类型供应商生效） + enableOpenaiResponsesWebsocket: z.boolean().optional(), + // 高并发模式（可选） + enableHighConcurrencyMode: z.boolean().optional(), + // 可选拦截 Anthropic Warmup 请求（可选） + interceptAnthropicWarmupRequests: z.boolean().optional(), + // thinking signature 整流器（可选） + enableThinkingSignatureRectifier: z.boolean().optional(), + // thinking budget 整流器（可选） + enableThinkingBudgetRectifier: z.boolean().optional(), + // thinking effort 冲突整流器（可选） + enableThinkingEffortConflictRectifier: z.boolean().optional(), + // Gemini function id 整流器（可选） + enableGeminiFunctionIdRectifier: z.boolean().optional(), + // billing header 整流器（可选） + enableBillingHeaderRectifier: z.boolean().optional(), + // Response API input 整流器（可选） + enableResponseInputRectifier: z.boolean().optional(), + // 非对话端点跨供应商 fallback（可选） + allowNonConversationEndpointProviderFallback: z.boolean().optional(), + // Fake 流式输出白名单（可选）。空数组表示显式禁用；缺省 → 使用默认四个图像生成模型。 + fakeStreamingWhitelist: z + .array( + z.object({ + model: z + .string() + .min(1, "model 不能为空") + .max(200, "model 不能超过 200 个字符") + .transform((value) => value.trim()) + .refine((value) => value.length > 0, { message: "model 不能为空" }), + groupTags: z + .array( + z + .string() + .min(1) + .transform((value) => value.trim()) + .refine((value) => value.length > 0, { message: "groupTag 不能为空" }) + ) + .default([]) + .transform((tags) => Array.from(new Set(tags))), + }) + ) + .superRefine((entries, ctx) => { + const seen = new Set(); + for (let index = 0; index < entries.length; index += 1) { + const model = entries[index].model; + if (seen.has(model)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `fakeStreamingWhitelist 模型重复: ${model}`, + path: [index, "model"], + }); + } + seen.add(model); } - seen.add(model); - } - }) - .optional(), - // Codex Session ID 补全（可选） - enableCodexSessionIdCompletion: z.boolean().optional(), - // Claude metadata.user_id 注入（可选） - enableClaudeMetadataUserIdInjection: z.boolean().optional(), - // 响应整流（可选） - enableResponseFixer: z.boolean().optional(), - responseFixerConfig: z - .object({ - fixTruncatedJson: z.boolean().optional(), - fixSseFormat: z.boolean().optional(), - fixEncoding: z.boolean().optional(), - maxJsonDepth: z.coerce.number().int("maxJsonDepth 必须是整数").min(1).max(2000).optional(), - maxFixSize: z.coerce - .number() - .int("maxFixSize 必须是整数") - .min(1024) - .max(10 * 1024 * 1024) - .optional(), - }) - .partial() - .optional(), + }) + .optional(), + // Codex Session ID 补全（可选） + enableCodexSessionIdCompletion: z.boolean().optional(), + // Claude metadata.user_id 注入（可选） + enableClaudeMetadataUserIdInjection: z.boolean().optional(), + // 响应整流（可选） + enableResponseFixer: z.boolean().optional(), + responseFixerConfig: z + .object({ + fixTruncatedJson: z.boolean().optional(), + fixSseFormat: z.boolean().optional(), + fixEncoding: z.boolean().optional(), + maxJsonDepth: z.coerce.number().int("maxJsonDepth 必须是整数").min(1).max(2000).optional(), + maxFixSize: z.coerce + .number() + .int("maxFixSize 必须是整数") + .min(1024) + .max(10 * 1024 * 1024) + .optional(), + }) + .partial() + .optional(), - // Quota lease settings - quotaDbRefreshIntervalSeconds: z.coerce - .number() - .int("DB refresh interval must be an integer") - .min(1, "DB refresh interval cannot be less than 1 second") - .max(300, "DB refresh interval cannot exceed 300 seconds") - .optional(), - quotaLeasePercent5h: z.coerce - .number() - .min(0, "Lease percent cannot be negative") - .max(1, "Lease percent cannot exceed 1") - .optional(), - quotaLeasePercentDaily: z.coerce - .number() - .min(0, "Lease percent cannot be negative") - .max(1, "Lease percent cannot exceed 1") - .optional(), - quotaLeasePercentWeekly: z.coerce - .number() - .min(0, "Lease percent cannot be negative") - .max(1, "Lease percent cannot exceed 1") - .optional(), - quotaLeasePercentMonthly: z.coerce - .number() - .min(0, "Lease percent cannot be negative") - .max(1, "Lease percent cannot exceed 1") - .optional(), - quotaLeaseCapUsd: z.coerce.number().min(0, "Lease cap cannot be negative").nullable().optional(), - publicStatusWindowHours: z.coerce - .number() - .int("PUBLIC_STATUS_WINDOW_INVALID_INT") - .min(1, "PUBLIC_STATUS_WINDOW_TOO_SMALL") - .max(MAX_PUBLIC_STATUS_RANGE_HOURS, "PUBLIC_STATUS_WINDOW_TOO_LARGE") - .optional(), - publicStatusAggregationIntervalMinutes: z.coerce - .number() - .int("PUBLIC_STATUS_INTERVAL_INVALID_INT") - .refine( - (value) => - PUBLIC_STATUS_INTERVAL_OPTIONS.includes( - value as (typeof PUBLIC_STATUS_INTERVAL_OPTIONS)[number] - ), - { - message: "PUBLIC_STATUS_INTERVAL_INVALID", - } - ) - .optional(), + // Quota lease settings + quotaDbRefreshIntervalSeconds: z.coerce + .number() + .int("DB refresh interval must be an integer") + .min(1, "DB refresh interval cannot be less than 1 second") + .max(300, "DB refresh interval cannot exceed 300 seconds") + .optional(), + quotaLeasePercent5h: z.coerce + .number() + .min(0, "Lease percent cannot be negative") + .max(1, "Lease percent cannot exceed 1") + .optional(), + quotaLeasePercentDaily: z.coerce + .number() + .min(0, "Lease percent cannot be negative") + .max(1, "Lease percent cannot exceed 1") + .optional(), + quotaLeasePercentWeekly: z.coerce + .number() + .min(0, "Lease percent cannot be negative") + .max(1, "Lease percent cannot exceed 1") + .optional(), + quotaLeasePercentMonthly: z.coerce + .number() + .min(0, "Lease percent cannot be negative") + .max(1, "Lease percent cannot exceed 1") + .optional(), + quotaLeaseCapUsd: z.coerce + .number() + .min(0, "Lease cap cannot be negative") + .nullable() + .optional(), + publicStatusWindowHours: z.coerce + .number() + .int("PUBLIC_STATUS_WINDOW_INVALID_INT") + .min(1, "PUBLIC_STATUS_WINDOW_TOO_SMALL") + .max(MAX_PUBLIC_STATUS_RANGE_HOURS, "PUBLIC_STATUS_WINDOW_TOO_LARGE") + .optional(), + publicStatusAggregationIntervalMinutes: z.coerce + .number() + .int("PUBLIC_STATUS_INTERVAL_INVALID_INT") + .refine( + (value) => + PUBLIC_STATUS_INTERVAL_OPTIONS.includes( + value as (typeof PUBLIC_STATUS_INTERVAL_OPTIONS)[number] + ), + { + message: "PUBLIC_STATUS_INTERVAL_INVALID", + } + ) + .optional(), - // 客户端 IP 提取链（可选；null 表示使用内置默认） - ipExtractionConfig: z - .union([ - z.null(), - z.object({ - headers: z.array( - z.object({ - name: z.string(), - pick: XFF_PICK_SCHEMA.optional(), - }) - ), - }), - ]) - .optional(), - // 是否启用 IP 归属地查询（可选） - ipGeoLookupEnabled: z.boolean().optional(), -}); + // 客户端 IP 提取链（可选；null 表示使用内置默认） + ipExtractionConfig: z + .union([ + z.null(), + z.object({ + headers: z.array( + z.object({ + name: z.string(), + pick: XFF_PICK_SCHEMA.optional(), + }) + ), + }), + ]) + .optional(), + // 是否启用 IP 归属地查询（可选） + ipGeoLookupEnabled: z.boolean().optional(), + }) + .superRefine((data, ctx) => { + const values = [ + data.racingTotalTimeoutMs, + data.stickySlaMs, + data.maxDiscoveryRounds, + data.discoverySlaMs, + ]; + if (values.every((value) => value !== undefined)) { + const requiredWindow = data.stickySlaMs! + data.maxDiscoveryRounds! * data.discoverySlaMs!; + if (data.racingTotalTimeoutMs! < requiredWindow) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["racingTotalTimeoutMs"], + message: "竞速总超时必须不小于 Sticky SLA + Discovery 轮数 × Discovery SLA", + }); + } + } + }); // 导出类型推断 diff --git a/src/repository/_shared/transformers.ts b/src/repository/_shared/transformers.ts index 51fc05202..568a50ba7 100644 --- a/src/repository/_shared/transformers.ts +++ b/src/repository/_shared/transformers.ts @@ -298,6 +298,13 @@ export function toSystemSettings(dbSettings: any): SystemSettings { quotaLeaseCapUsd: dbSettings?.quotaLeaseCapUsd ? parseFloat(dbSettings.quotaLeaseCapUsd) : null, publicStatusWindowHours: dbSettings?.publicStatusWindowHours ?? 24, publicStatusAggregationIntervalMinutes: dbSettings?.publicStatusAggregationIntervalMinutes ?? 5, + discoveryEnabled: dbSettings?.discoveryEnabled ?? false, + discoveryConcurrency: dbSettings?.discoveryConcurrency ?? 2, + maxDiscoveryRounds: dbSettings?.maxDiscoveryRounds ?? 2, + discoverySlaMs: dbSettings?.discoverySlaMs ?? 10_000, + stickySlaMs: dbSettings?.stickySlaMs ?? 20_000, + racingTotalTimeoutMs: dbSettings?.racingTotalTimeoutMs ?? 60_000, + stickyTimeoutCooldownMs: dbSettings?.stickyTimeoutCooldownMs ?? 300_000, ipExtractionConfig: dbSettings?.ipExtractionConfig ?? null, ipGeoLookupEnabled: dbSettings?.ipGeoLookupEnabled ?? true, createdAt: dbSettings?.createdAt ? new Date(dbSettings.createdAt) : new Date(), diff --git a/src/repository/system-config.ts b/src/repository/system-config.ts index 20a53bc3b..1e6ba0943 100644 --- a/src/repository/system-config.ts +++ b/src/repository/system-config.ts @@ -191,6 +191,13 @@ function createFallbackSettings(): SystemSettings { quotaLeaseCapUsd: null, publicStatusWindowHours: 24, publicStatusAggregationIntervalMinutes: 5, + discoveryEnabled: false, + discoveryConcurrency: 2, + maxDiscoveryRounds: 2, + discoverySlaMs: 10_000, + stickySlaMs: 20_000, + racingTotalTimeoutMs: 60_000, + stickyTimeoutCooldownMs: 300_000, ipExtractionConfig: null, ipGeoLookupEnabled: true, createdAt: now, @@ -247,6 +254,13 @@ const BASE_SETTINGS_COLUMNS: SettingsSelection = { quotaLeaseCapUsd: systemSettings.quotaLeaseCapUsd, publicStatusWindowHours: systemSettings.publicStatusWindowHours, publicStatusAggregationIntervalMinutes: systemSettings.publicStatusAggregationIntervalMinutes, + discoveryEnabled: systemSettings.discoveryEnabled, + discoveryConcurrency: systemSettings.discoveryConcurrency, + maxDiscoveryRounds: systemSettings.maxDiscoveryRounds, + discoverySlaMs: systemSettings.discoverySlaMs, + stickySlaMs: systemSettings.stickySlaMs, + racingTotalTimeoutMs: systemSettings.racingTotalTimeoutMs, + stickyTimeoutCooldownMs: systemSettings.stickyTimeoutCooldownMs, createdAt: systemSettings.createdAt, updatedAt: systemSettings.updatedAt, passThroughUpstreamErrorMessage: systemSettings.passThroughUpstreamErrorMessage, @@ -264,6 +278,48 @@ const RECENT_COLUMN_LADDER: ReadonlyArray<{ // 本层更新失败（仍有列缺失）时记录的告警 updateWarn: string; }> = [ + { + key: "stickyTimeoutCooldownMs", + column: systemSettings.stickyTimeoutCooldownMs, + selectWarn: "system_settings 缺少 stickyTimeoutCooldownMs，回退到上一代字段集。", + updateWarn: "system_settings 缺少 stickyTimeoutCooldownMs，回退到上一代字段集。", + }, + { + key: "racingTotalTimeoutMs", + column: systemSettings.racingTotalTimeoutMs, + selectWarn: "system_settings 缺少 racingTotalTimeoutMs，回退到上一代字段集。", + updateWarn: "system_settings 缺少 racingTotalTimeoutMs，回退到上一代字段集。", + }, + { + key: "stickySlaMs", + column: systemSettings.stickySlaMs, + selectWarn: "system_settings 缺少 stickySlaMs，回退到上一代字段集。", + updateWarn: "system_settings 缺少 stickySlaMs，回退到上一代字段集。", + }, + { + key: "discoverySlaMs", + column: systemSettings.discoverySlaMs, + selectWarn: "system_settings 缺少 discoverySlaMs，回退到上一代字段集。", + updateWarn: "system_settings 缺少 discoverySlaMs，回退到上一代字段集。", + }, + { + key: "maxDiscoveryRounds", + column: systemSettings.maxDiscoveryRounds, + selectWarn: "system_settings 缺少 maxDiscoveryRounds，回退到上一代字段集。", + updateWarn: "system_settings 缺少 maxDiscoveryRounds，回退到上一代字段集。", + }, + { + key: "discoveryConcurrency", + column: systemSettings.discoveryConcurrency, + selectWarn: "system_settings 缺少 discoveryConcurrency，回退到上一代字段集。", + updateWarn: "system_settings 缺少 discoveryConcurrency，回退到上一代字段集。", + }, + { + key: "discoveryEnabled", + column: systemSettings.discoveryEnabled, + selectWarn: "system_settings 缺少 discoveryEnabled，回退到上一代字段集。", + updateWarn: "system_settings 缺少 discoveryEnabled，回退到上一代字段集。", + }, { key: "enableGeminiFunctionIdRectifier", column: systemSettings.enableGeminiFunctionIdRectifier, @@ -620,6 +676,28 @@ export async function updateSystemSettings( updates.billHedgeLosers = payload.billHedgeLosers; } + if (payload.discoveryEnabled !== undefined) { + updates.discoveryEnabled = payload.discoveryEnabled; + } + if (payload.discoveryConcurrency !== undefined) { + updates.discoveryConcurrency = payload.discoveryConcurrency; + } + if (payload.maxDiscoveryRounds !== undefined) { + updates.maxDiscoveryRounds = payload.maxDiscoveryRounds; + } + if (payload.discoverySlaMs !== undefined) { + updates.discoverySlaMs = payload.discoverySlaMs; + } + if (payload.stickySlaMs !== undefined) { + updates.stickySlaMs = payload.stickySlaMs; + } + if (payload.racingTotalTimeoutMs !== undefined) { + updates.racingTotalTimeoutMs = payload.racingTotalTimeoutMs; + } + if (payload.stickyTimeoutCooldownMs !== undefined) { + updates.stickyTimeoutCooldownMs = payload.stickyTimeoutCooldownMs; + } + // 系统时区配置字段（如果提供） if (payload.timezone !== undefined) { updates.timezone = payload.timezone; diff --git a/src/types/system-config.ts b/src/types/system-config.ts index 16ef36a98..dd47b7b06 100644 --- a/src/types/system-config.ts +++ b/src/types/system-config.ts @@ -146,14 +146,14 @@ export interface SystemSettings { publicStatusWindowHours: number; publicStatusAggregationIntervalMinutes: number; - /** Bounded streaming Discovery (PR2; persisted/configured in PR3). */ - discoveryEnabled?: boolean; - discoveryConcurrency?: number; - maxDiscoveryRounds?: number; - discoverySlaMs?: number; - stickySlaMs?: number; - racingTotalTimeoutMs?: number; - stickyTimeoutCooldownMs?: number; + /** Bounded streaming Discovery settings. */ + discoveryEnabled: boolean; + discoveryConcurrency: number; + maxDiscoveryRounds: number; + discoverySlaMs: number; + stickySlaMs: number; + racingTotalTimeoutMs: number; + stickyTimeoutCooldownMs: number; createdAt: Date; updatedAt: Date; diff --git a/tests/unit/repository/system-config-degradation-ladder.test.ts b/tests/unit/repository/system-config-degradation-ladder.test.ts index e274a37ec..54020cdb6 100644 --- a/tests/unit/repository/system-config-degradation-ladder.test.ts +++ b/tests/unit/repository/system-config-degradation-ladder.test.ts @@ -7,6 +7,13 @@ import type { UpdateSystemSettingsInput } from "@/types/system-config"; // 近代新增列（最新在前），降级链按引入顺序逐层累计剥离。 const RECENT_COLUMNS = [ + "stickyTimeoutCooldownMs", + "racingTotalTimeoutMs", + "stickySlaMs", + "discoverySlaMs", + "maxDiscoveryRounds", + "discoveryConcurrency", + "discoveryEnabled", "enableGeminiFunctionIdRectifier", "enableThinkingEffortConflictRectifier", "billHedgeLosers", @@ -18,6 +25,13 @@ const RECENT_COLUMNS = [ // 全量字段集（44 列）。 const FULL_COLUMNS = [ + "discoveryEnabled", + "discoveryConcurrency", + "maxDiscoveryRounds", + "discoverySlaMs", + "stickySlaMs", + "racingTotalTimeoutMs", + "stickyTimeoutCooldownMs", "enableGeminiFunctionIdRectifier", "billHedgeLosers", "billNonSuccessfulRequests", @@ -167,7 +181,7 @@ describe("SystemSettings：列降级阶梯的尝试序列锁定", () => { const selectMock = vi.fn((selection: Record) => { selections.push(sortedKeys(selection)); callIndex += 1; - if (callIndex < 9) { + if (callIndex < 16) { return createRejectingSelectQuery({ code: "42703" }); } return createResolvingSelectQuery([ @@ -200,14 +214,14 @@ describe("SystemSettings：列降级阶梯的尝试序列锁定", () => { const result = await getSystemSettings(); - expect(selectMock).toHaveBeenCalledTimes(9); - // 第 8 次（近代链末层）不含这两列；第 9 次（passThrough 世代）重新包含。 - expect(selections[7]).not.toContain("enableThinkingEffortConflictRectifier"); - expect(selections[7]).not.toContain("allowNonConversationEndpointProviderFallback"); - expect(selections[7]).toContain("passThroughUpstreamErrorMessage"); - expect(selections[8]).toContain("enableThinkingEffortConflictRectifier"); - expect(selections[8]).toContain("allowNonConversationEndpointProviderFallback"); - expect(selections[8]).not.toContain("passThroughUpstreamErrorMessage"); + expect(selectMock).toHaveBeenCalledTimes(16); + // 第 15 次（近代链末层）不含这些新列；第 16 次（passThrough 世代）重新包含旧列。 + expect(selections[14]).not.toContain("enableThinkingEffortConflictRectifier"); + expect(selections[14]).not.toContain("allowNonConversationEndpointProviderFallback"); + expect(selections[14]).toContain("passThroughUpstreamErrorMessage"); + expect(selections[15]).toContain("enableThinkingEffortConflictRectifier"); + expect(selections[15]).toContain("allowNonConversationEndpointProviderFallback"); + expect(selections[15]).not.toContain("passThroughUpstreamErrorMessage"); // 世代字段集选出的真实值要透传，缺失列由 transformer 落默认值。 expect(result.siteTitle).toBe("Era Row"); @@ -285,7 +299,7 @@ describe("SystemSettings：列降级阶梯的尝试序列锁定", () => { "system_settings 表列缺失，请执行数据库迁移以升级数据库结构。" ); - expect(updateMock).toHaveBeenCalledTimes(11); + expect(updateMock).toHaveBeenCalledTimes(18); const expectedReturningSequence = [ [...FULL_COLUMNS], diff --git a/tests/unit/repository/system-config-update-missing-columns.test.ts b/tests/unit/repository/system-config-update-missing-columns.test.ts index 1571b0f9c..611ab919d 100644 --- a/tests/unit/repository/system-config-update-missing-columns.test.ts +++ b/tests/unit/repository/system-config-update-missing-columns.test.ts @@ -299,7 +299,7 @@ describe("SystemSettings：数据库缺列时的保存兜底", () => { vi.setSystemTime(now); // 第一次 select(fullSelection) 因新列缺失而抛 42703； - // 第二次 select(selectionWithoutGeminiFunctionId) 命中——验证新列已加入降级链最外层。 + // 第二次 select(selectionWithoutDiscoveryColumn) 命中——验证新列已加入降级链最外层。 const selectMock = vi .fn() .mockReturnValueOnce(createRejectedThenableQuery({ code: "42703" })) @@ -339,11 +339,11 @@ describe("SystemSettings：数据库缺列时的保存兜底", () => { expect(result.siteTitle).toBe("Claude Code Hub"); expect(result.enableHttp2).toBe(true); - // 关键回归保护：第二次 select 必须恰好剥离了新列（最外层降级）， - // 而非旧行为先剥离 enableThinkingEffortConflictRectifier。若新列未加入降级链最外层，下面两条断言会失败。 + // 关键回归保护：第二次 select 必须恰好剥离了最新 Discovery 列， + // 而非旧行为先剥离 enableGeminiFunctionIdRectifier。 const secondSelection = selectMock.mock.calls[1]?.[0] as Record; - expect(secondSelection).not.toHaveProperty("enableGeminiFunctionIdRectifier"); - expect(secondSelection).toHaveProperty("enableThinkingEffortConflictRectifier"); + expect(secondSelection).not.toHaveProperty("stickyTimeoutCooldownMs"); + expect(secondSelection).toHaveProperty("racingTotalTimeoutMs"); vi.useRealTimers(); }); diff --git a/tests/unit/validation/system-settings-discovery.test.ts b/tests/unit/validation/system-settings-discovery.test.ts new file mode 100644 index 000000000..f6b7df030 --- /dev/null +++ b/tests/unit/validation/system-settings-discovery.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { UpdateSystemSettingsSchema } from "@/lib/validation/schemas"; + +describe("UpdateSystemSettingsSchema Discovery settings", () => { + it("accepts the recommended defaults", () => { + const result = UpdateSystemSettingsSchema.parse({ + discoveryEnabled: false, + discoveryConcurrency: 2, + maxDiscoveryRounds: 2, + discoverySlaMs: 10_000, + stickySlaMs: 20_000, + racingTotalTimeoutMs: 60_000, + stickyTimeoutCooldownMs: 300_000, + }); + expect(result.discoveryConcurrency).toBe(2); + }); + + it("rejects a total deadline shorter than the configured discovery window", () => { + expect(() => + UpdateSystemSettingsSchema.parse({ + discoverySlaMs: 10_000, + stickySlaMs: 20_000, + maxDiscoveryRounds: 2, + racingTotalTimeoutMs: 30_000, + }) + ).toThrow("竞速总超时"); + }); + + it("allows partial updates so the server can merge them with stored settings", () => { + expect(UpdateSystemSettingsSchema.parse({ discoveryEnabled: true })).toEqual({ + discoveryEnabled: true, + }); + }); +}); From d4cba72c6206fc01d723e94221a4a94635a92854 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 14:54:02 -0400 Subject: [PATCH 09/89] fix(proxy): close discovery coordinator lifecycle gaps --- .../v1/_lib/proxy/discovery-coordinator.ts | 45 ++++- src/app/v1/_lib/proxy/discovery-validity.ts | 51 +++++- src/app/v1/_lib/proxy/forwarder.ts | 154 +++++++++++++----- .../unit/proxy/discovery-coordinator.test.ts | 22 ++- tests/unit/proxy/discovery-validity.test.ts | 14 ++ 5 files changed, 227 insertions(+), 59 deletions(-) diff --git a/src/app/v1/_lib/proxy/discovery-coordinator.ts b/src/app/v1/_lib/proxy/discovery-coordinator.ts index 44d9de404..4d4462b2c 100644 --- a/src/app/v1/_lib/proxy/discovery-coordinator.ts +++ b/src/app/v1/_lib/proxy/discovery-coordinator.ts @@ -29,8 +29,13 @@ export type DiscoveryAttempt = { export type DiscoveryAction = | { type: "commit_normal"; attemptId: string } | { type: "promote_fallback"; attemptId: string } - | { type: "cancel"; attemptIds: string[] } - | { type: "launch"; slots: number } + | { type: "cancel"; attemptIds: string[]; promoteAttemptId?: string } + | { + type: "launch"; + slots: number; + cancelAttemptIds?: string[]; + promoteAttemptId?: string; + } | { type: "none" } | { type: "terminal_failure" }; @@ -63,6 +68,7 @@ export class DiscoveryCoordinator { beginRound(): { requestEpoch: number; roundEpoch: number; round: number } { this.roundEpoch += 1; + if (!this.isTerminal) this.state = "DISCOVERY_RACING"; return { ...this.epochs, round: this.round }; } @@ -102,6 +108,17 @@ export class DiscoveryCoordinator { const attempt = this.attempts.get(id); if (!attempt?.pending) return { type: "none" }; attempt.ready = true; + if (attempt.kind === "fallback") { + const pendingNormal = Array.from(this.attempts.values()).some( + (candidate) => candidate.pending && candidate.kind === "normal" + ); + if (!pendingNormal) { + attempt.pending = false; + this.state = "FALLBACK_ACTIVE"; + return { type: "promote_fallback", attemptId: attempt.id }; + } + return { type: "none" }; + } return this.chooseReadyNormal(); } @@ -168,14 +185,19 @@ export class DiscoveryCoordinator { .filter((attempt) => attempt.pending && attempt.kind === "normal") .sort(compareAttempts); if (currentFallback && pendingNormal.length > 0) { + const cancelAttemptIds = pendingNormal.map((attempt) => attempt.id); for (const attempt of pendingNormal) attempt.pending = false; if (this.round < this.maxRounds) { this.round += 1; this.roundEpoch += 1; this.state = "DISCOVERY_RACING"; - return { type: "launch", slots: Math.max(1, this.concurrency - 1) }; + return { + type: "launch", + slots: Math.max(1, this.concurrency - 1), + cancelAttemptIds, + }; } - return { type: "none" }; + return { type: "cancel", attemptIds: cancelAttemptIds }; } if (pendingNormal.length === 0) { if (currentFallback) { @@ -191,11 +213,18 @@ export class DiscoveryCoordinator { const losers = pendingNormal.slice(1).map((attempt) => attempt.id); for (const id of losers) this.attempts.get(id)!.pending = false; - if (currentFallback) { - currentFallback.pending = true; - return { type: "cancel", attemptIds: losers }; + if (this.round < this.maxRounds) { + this.round += 1; + this.roundEpoch += 1; + this.state = "DISCOVERY_RACING"; + return { + type: "launch", + slots: Math.max(1, this.concurrency - 1), + cancelAttemptIds: losers, + promoteAttemptId: fallback.id, + }; } - return { type: "cancel", attemptIds: losers }; + return { type: "cancel", attemptIds: losers, promoteAttemptId: fallback.id }; } onDeadline(): DiscoveryAction { diff --git a/src/app/v1/_lib/proxy/discovery-validity.ts b/src/app/v1/_lib/proxy/discovery-validity.ts index 29ad6a960..d4ece4eff 100644 --- a/src/app/v1/_lib/proxy/discovery-validity.ts +++ b/src/app/v1/_lib/proxy/discovery-validity.ts @@ -137,11 +137,58 @@ export class DiscoveryValidityParser { push(chunk: Uint8Array | string): DiscoveryValidity { this.buffered += typeof chunk === "string" ? chunk : this.decoder.decode(chunk, { stream: true }); - const result = classifyDiscoveryChunk(this.buffered, this.protocol); + + // SSE streams are line framed. Consume each completed line once instead + // of reparsing the complete prefix on every chunk (which is quadratic on + // long streams). Keep only the unfinished line for the next push. + if (this.buffered.includes("\n")) { + const lines = this.buffered.split(/\r?\n/); + this.buffered = lines.pop() ?? ""; + for (const line of lines) this.consumeLine(line); + } + + // Some providers return one raw JSON object without an SSE newline. Parse + // it only when the complete object is available; incomplete JSON remains + // buffered and is not repeatedly scanned as a protocol event. + const tail = this.buffered.trim(); + if (tail) { + const candidate = tail.startsWith("data:") ? tail.slice(5).trim() : tail; + if (candidate === "[DONE]") { + this._terminal = true; + this.buffered = ""; + } else if (candidate.startsWith("{") || candidate.startsWith("[")) { + try { + const value = JSON.parse(candidate) as unknown; + this.consumeValue(value); + this.buffered = ""; + } catch { + // Keep incomplete raw JSON until the next chunk completes it. + } + } + } + + return { ready: this._ready && !this._error, terminal: this._terminal, error: this._error }; + } + + private consumeLine(line: string): void { + const candidate = line.startsWith("data:") ? line.slice(5).trim() : line.trim(); + if (!candidate || candidate.startsWith(":")) return; + if (candidate === "[DONE]") { + this._terminal = true; + return; + } + try { + this.consumeValue(JSON.parse(candidate) as unknown); + } catch { + // Ignore comments and incomplete/non-JSON protocol lines. + } + } + + private consumeValue(value: unknown): void { + const result = classifyJson(value, this.protocol); this._ready ||= result.ready; this._terminal ||= result.terminal; this._error ||= result.error; - return { ready: this._ready && !this._error, terminal: this._terminal, error: this._error }; } get ready(): boolean { diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 122e00d71..18182a2e1 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -63,7 +63,7 @@ import { buildProxyUrl } from "../url"; import { rectifyBillingHeader } from "./billing-header-rectifier"; import { bindClientAbortListener } from "./client-abort-listener"; import { deriveClientSafeUpstreamErrorMessage } from "./client-error-message"; -import { DiscoveryCoordinator } from "./discovery-coordinator"; +import { type DiscoveryAction, DiscoveryCoordinator } from "./discovery-coordinator"; import { type DiscoveryProtocol, DiscoveryValidityParser } from "./discovery-validity"; import { isStandardProxyEndpointPath } from "./endpoint-family-catalog"; import { resolveEndpointPolicy, shouldEnforceStrictEndpointPoolPolicy } from "./endpoint-policy"; @@ -4875,6 +4875,7 @@ export class ProxyForwarder { pending: boolean; ready: boolean; round: number; + readerTransferred: boolean; } >(); const launched = new Set(); @@ -4889,6 +4890,7 @@ export class ProxyForwarder { let totalTimer: NodeJS.Timeout | null = null; let roundTimer: NodeJS.Timeout | null = null; let stickyTimer: NodeJS.Timeout | null = null; + let executeCoordinatorAction: (action: DiscoveryAction) => Promise = async () => {}; let resolveResult: ((result: { response?: Response; error?: Error }) => void) | null = null; const resultPromise = new Promise<{ response?: Response; error?: Error }>((resolve) => { resolveResult = resolve; @@ -4904,6 +4906,7 @@ export class ProxyForwarder { }; const cancelAttempt = (attempt: (typeof winner & { id: string }) | null, reason: string) => { + if (attempt?.readerTransferred) return; if (!attempt?.pending) return; attempt.pending = false; try { @@ -4911,7 +4914,12 @@ export class ProxyForwarder { } catch { /* abort is best effort */ } - void attempt.reader?.cancel(reason).catch(() => undefined); + try { + const cancelPromise = attempt.reader?.cancel(reason); + cancelPromise?.catch(() => undefined); + } catch (error) { + logger.debug("[Discovery] Reader cancel failed", { reason, error }); + } try { attempt.releaseAgent?.(); } catch { @@ -4933,8 +4941,18 @@ export class ProxyForwarder { if (roundTimer) clearTimeout(roundTimer); if (stickyTimer) clearTimeout(stickyTimer); cancelLosers(); - const attempted = new Set(launched); - await ProxyForwarder.clearSessionProviderBindings(session, attempted); + if (bindingSnapshot) { + if (bindingSnapshot.providerId != null) { + await SessionManager.clearVersionedSessionProvider( + bindingSnapshot, + bindingSnapshot.providerId, + 0 + ); + } + } else { + const attempted = new Set(launched); + await ProxyForwarder.clearSessionProviderBindings(session, attempted); + } resolveResult?.({ error }); }; @@ -4943,6 +4961,9 @@ export class ProxyForwarder { committed = true; winner = attempt; attempt.pending = false; + // From this point ResponseHandler owns the reader and agent release. + // No coordinator/timer path may cancel or release this attempt again. + attempt.readerTransferred = true; if (totalTimer) clearTimeout(totalTimer); if (roundTimer) clearTimeout(roundTimer); if (stickyTimer) clearTimeout(stickyTimer); @@ -5044,6 +5065,7 @@ export class ProxyForwarder { pending: true, ready: false, round: currentRound, + readerTransferred: false, provider, session: attemptSession, baseUrl: endpoint.baseUrl, @@ -5080,6 +5102,7 @@ export class ProxyForwarder { pending: boolean; ready: boolean; round: number; + readerTransferred: boolean; }; attempts.set(id, attempt); coordinator.addAttempt({ @@ -5138,7 +5161,7 @@ export class ProxyForwarder { .catch(async (error) => { if (committed || settled || !attempt.pending) return; attempt.pending = false; - coordinator.markFailed(id); + const failureAction = coordinator.markFailed(id); lastError = error instanceof Error ? error : new Error(String(error)); lastErrorCategory = await categorizeErrorAsync(lastError); session.addProviderToChain(provider, { @@ -5156,8 +5179,17 @@ export class ProxyForwarder { } attempt.releaseAgent?.(); releaseProviderRef(provider.id); - const replacement = await chooseCandidate(); - if (replacement && !committed && !settled) await launch(replacement, "normal"); + const actionOwnsNextStep = + failureAction.type === "promote_fallback" || + failureAction.type === "launch" || + failureAction.type === "terminal_failure"; + if (actionOwnsNextStep) { + await executeCoordinatorAction(failureAction); + } + if (!actionOwnsNextStep && !committed && !settled) { + const replacement = await chooseCandidate(); + if (replacement) await launch(replacement, "normal"); + } if ( Array.from(attempts.values()).every((candidate) => !candidate.pending) && noMoreCandidates @@ -5166,6 +5198,12 @@ export class ProxyForwarder { ProxyForwarder.resolveHedgeTerminalError(lastError, lastErrorCategory) ); } + }) + .catch((error) => { + logger.warn("[Discovery] Attempt completion handler failed", { + providerId: provider.id, + error, + }); }); }; @@ -5184,60 +5222,75 @@ export class ProxyForwarder { } } if (!committed && !settled) { - roundTimer = setTimeout(() => void onBoundary(), discoverySlaMs); + roundTimer = setTimeout(() => { + void onBoundary().catch((error) => + logger.warn("[Discovery] Round boundary failed", { error }) + ); + }, discoverySlaMs); } }; - const onBoundary = async () => { + executeCoordinatorAction = async (action) => { if (settled || committed) return; - const fallback = Array.from(attempts.values()).find( - (attempt) => attempt.pending && attempt.kind === "fallback" - ); - const pendingNormals = Array.from(attempts.values()) - .filter((attempt) => attempt.pending && attempt.kind === "normal") - .sort( - (a, b) => - (a.provider.priority || 0) - (b.provider.priority || 0) || a.sequence - b.sequence - ); - const readyNormal = pendingNormals.find((attempt) => attempt.ready); - if (readyNormal) { - await commit(readyNormal); + if (action.type === "cancel" || action.type === "launch") { + const cancelIds = + action.type === "cancel" ? action.attemptIds : (action.cancelAttemptIds ?? []); + for (const id of cancelIds) { + const attempt = attempts.get(id); + // Coordinator marks cancelled attempts non-pending before returning + // the action. Restore the transport-facing state long enough for the + // exactly-once cancellation/release path to run. + if (attempt && !attempt.readerTransferred) attempt.pending = true; + if (attempt) cancelAttempt(attempt, "discovery_round_boundary"); + } + if (action.promoteAttemptId) { + const fallback = attempts.get(action.promoteAttemptId); + if (fallback) fallback.kind = "fallback"; + if (action.type === "cancel" && currentRound < maxRounds) { + await launchNextRound(); + } + } + } + if (action.type === "commit_normal" || action.type === "promote_fallback") { + const attempt = attempts.get(action.attemptId); + if (attempt) await commit(attempt); return; } - // The fallback is held during the SLA window, but at the round - // boundary it may take over when no normal result is ready. - if (fallback?.ready) { - await commit(fallback); + if (action.type === "launch") { + await launchNextRound(); return; } - if (pendingNormals[0]) { - if (fallback) { - for (const loser of pendingNormals) cancelAttempt(loser, "discovery_round_boundary"); - if (currentRound < maxRounds) await launchNextRound(); - return; - } - pendingNormals[0].kind = "fallback"; - for (const loser of pendingNormals.slice(1)) - cancelAttempt(loser, "discovery_round_boundary"); - if (currentRound < maxRounds) await launchNextRound(); + if (action.type === "terminal_failure") { + await settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); return; } - if (currentRound < maxRounds) await launchNextRound(); - else await settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); + if (action.type === "none") { + const fallbackPending = Array.from(attempts.values()).some( + (attempt) => attempt.pending && attempt.kind === "fallback" + ); + if (fallbackPending && currentRound < maxRounds) { + await launchNextRound(); + } + } + }; + + const onBoundary = async () => { + if (settled || committed) return; + await executeCoordinatorAction(coordinator.onRoundBoundary()); }; const cleanupAbort = bindClientAbortListener(session.clientAbortSignal, () => { if (settled || committed) return; - void settleFailure(new ProxyError("Request aborted by client", 499, undefined, true)); + void settleFailure(new ProxyError("Request aborted by client", 499, undefined, true)).catch( + (error) => logger.warn("[Discovery] Client abort cleanup failed", { error }) + ); }); totalTimer = setTimeout(() => { if (settled || committed) return; - const fallback = Array.from(attempts.values()).find( - (attempt) => attempt.pending && attempt.kind === "fallback" && attempt.ready + void executeCoordinatorAction(coordinator.onDeadline()).catch((error) => + logger.warn("[Discovery] Deadline action failed", { error }) ); - if (fallback) void commit(fallback); - else void settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); }, totalTimeoutMs); try { @@ -5267,8 +5320,15 @@ export class ProxyForwarder { logger.debug("[Discovery] Failed to clear timed-out Sticky", { error }) ); } - if (currentRound < maxRounds) void launchNextRound(); - else void onBoundary(); + if (currentRound < maxRounds) { + void launchNextRound().catch((error) => + logger.warn("[Discovery] Sticky round launch failed", { error }) + ); + } else { + void onBoundary().catch((error) => + logger.warn("[Discovery] Sticky boundary failed", { error }) + ); + } } }, stickySlaMs); } else { @@ -5285,7 +5345,11 @@ export class ProxyForwarder { noMoreCandidates = true; } } - roundTimer = setTimeout(() => void onBoundary(), discoverySlaMs); + roundTimer = setTimeout(() => { + void onBoundary().catch((error) => + logger.warn("[Discovery] Round boundary failed", { error }) + ); + }, discoverySlaMs); } const result = await resultPromise; if (result.error) throw result.error; diff --git a/tests/unit/proxy/discovery-coordinator.test.ts b/tests/unit/proxy/discovery-coordinator.test.ts index 70b09b6d0..35585c473 100644 --- a/tests/unit/proxy/discovery-coordinator.test.ts +++ b/tests/unit/proxy/discovery-coordinator.test.ts @@ -27,7 +27,11 @@ describe("DiscoveryCoordinator", () => { coordinator.addAttempt(attempt("a", 1)); coordinator.addAttempt(attempt("b", 2)); const action = coordinator.onRoundBoundary(); - expect(action.type).toBe("cancel"); + expect(action).toMatchObject({ + type: "launch", + promoteAttemptId: "a", + cancelAttemptIds: ["b"], + }); expect(coordinator.snapshot.find((item) => item.id === "a")?.kind).toBe("fallback"); expect(coordinator.snapshot.filter((item) => item.pending)).toHaveLength(1); }); @@ -45,9 +49,7 @@ describe("DiscoveryCoordinator", () => { it("promotes a ready fallback at the round boundary when no normal is ready", () => { const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); coordinator.addAttempt(attempt("a", 1, "fallback")); - coordinator.addAttempt(attempt("b", 1, "normal")); - expect(coordinator.markReady("a")).toEqual({ type: "none" }); - expect(coordinator.onRoundBoundary()).toEqual({ type: "promote_fallback", attemptId: "a" }); + expect(coordinator.markReady("a")).toEqual({ type: "promote_fallback", attemptId: "a" }); expect(coordinator.snapshot.find((item) => item.id === "a")?.kind).toBe("fallback"); }); @@ -60,4 +62,16 @@ describe("DiscoveryCoordinator", () => { coordinator.markReady("c"); expect(coordinator.onRoundBoundary()).toEqual({ type: "commit_normal", attemptId: "c" }); }); + + it("reports normal attempts cancelled when retaining an existing fallback", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 3 }); + coordinator.addAttempt(attempt("fallback", 1, "fallback")); + coordinator.addAttempt(attempt("normal", 2)); + const action = coordinator.onRoundBoundary(); + expect(action).toEqual({ + type: "launch", + slots: 1, + cancelAttemptIds: ["normal"], + }); + }); }); diff --git a/tests/unit/proxy/discovery-validity.test.ts b/tests/unit/proxy/discovery-validity.test.ts index 4d5e6cae2..83feb4f8e 100644 --- a/tests/unit/proxy/discovery-validity.test.ts +++ b/tests/unit/proxy/discovery-validity.test.ts @@ -60,4 +60,18 @@ describe("discovery validity", () => { ).ready ).toBe(true); }); + + it("consumes split SSE lines incrementally without waiting for the full stream", () => { + const parser = new DiscoveryValidityParser("openai-chat"); + expect(parser.push('data: {"choices":[{"delta":{"content":"hel')).toEqual({ + ready: false, + terminal: false, + error: false, + }); + expect(parser.push('lo"}}]}\n\n')).toEqual({ + ready: true, + terminal: false, + error: false, + }); + }); }); From 722900e3aebb3ad36b604a799eae45808d186754 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 14:54:02 -0400 Subject: [PATCH 10/89] fix(proxy): close discovery coordinator lifecycle gaps --- .../v1/_lib/proxy/discovery-coordinator.ts | 45 ++++- src/app/v1/_lib/proxy/discovery-validity.ts | 51 +++++- src/app/v1/_lib/proxy/forwarder.ts | 154 +++++++++++++----- .../unit/proxy/discovery-coordinator.test.ts | 22 ++- tests/unit/proxy/discovery-validity.test.ts | 14 ++ 5 files changed, 227 insertions(+), 59 deletions(-) diff --git a/src/app/v1/_lib/proxy/discovery-coordinator.ts b/src/app/v1/_lib/proxy/discovery-coordinator.ts index 44d9de404..4d4462b2c 100644 --- a/src/app/v1/_lib/proxy/discovery-coordinator.ts +++ b/src/app/v1/_lib/proxy/discovery-coordinator.ts @@ -29,8 +29,13 @@ export type DiscoveryAttempt = { export type DiscoveryAction = | { type: "commit_normal"; attemptId: string } | { type: "promote_fallback"; attemptId: string } - | { type: "cancel"; attemptIds: string[] } - | { type: "launch"; slots: number } + | { type: "cancel"; attemptIds: string[]; promoteAttemptId?: string } + | { + type: "launch"; + slots: number; + cancelAttemptIds?: string[]; + promoteAttemptId?: string; + } | { type: "none" } | { type: "terminal_failure" }; @@ -63,6 +68,7 @@ export class DiscoveryCoordinator { beginRound(): { requestEpoch: number; roundEpoch: number; round: number } { this.roundEpoch += 1; + if (!this.isTerminal) this.state = "DISCOVERY_RACING"; return { ...this.epochs, round: this.round }; } @@ -102,6 +108,17 @@ export class DiscoveryCoordinator { const attempt = this.attempts.get(id); if (!attempt?.pending) return { type: "none" }; attempt.ready = true; + if (attempt.kind === "fallback") { + const pendingNormal = Array.from(this.attempts.values()).some( + (candidate) => candidate.pending && candidate.kind === "normal" + ); + if (!pendingNormal) { + attempt.pending = false; + this.state = "FALLBACK_ACTIVE"; + return { type: "promote_fallback", attemptId: attempt.id }; + } + return { type: "none" }; + } return this.chooseReadyNormal(); } @@ -168,14 +185,19 @@ export class DiscoveryCoordinator { .filter((attempt) => attempt.pending && attempt.kind === "normal") .sort(compareAttempts); if (currentFallback && pendingNormal.length > 0) { + const cancelAttemptIds = pendingNormal.map((attempt) => attempt.id); for (const attempt of pendingNormal) attempt.pending = false; if (this.round < this.maxRounds) { this.round += 1; this.roundEpoch += 1; this.state = "DISCOVERY_RACING"; - return { type: "launch", slots: Math.max(1, this.concurrency - 1) }; + return { + type: "launch", + slots: Math.max(1, this.concurrency - 1), + cancelAttemptIds, + }; } - return { type: "none" }; + return { type: "cancel", attemptIds: cancelAttemptIds }; } if (pendingNormal.length === 0) { if (currentFallback) { @@ -191,11 +213,18 @@ export class DiscoveryCoordinator { const losers = pendingNormal.slice(1).map((attempt) => attempt.id); for (const id of losers) this.attempts.get(id)!.pending = false; - if (currentFallback) { - currentFallback.pending = true; - return { type: "cancel", attemptIds: losers }; + if (this.round < this.maxRounds) { + this.round += 1; + this.roundEpoch += 1; + this.state = "DISCOVERY_RACING"; + return { + type: "launch", + slots: Math.max(1, this.concurrency - 1), + cancelAttemptIds: losers, + promoteAttemptId: fallback.id, + }; } - return { type: "cancel", attemptIds: losers }; + return { type: "cancel", attemptIds: losers, promoteAttemptId: fallback.id }; } onDeadline(): DiscoveryAction { diff --git a/src/app/v1/_lib/proxy/discovery-validity.ts b/src/app/v1/_lib/proxy/discovery-validity.ts index 29ad6a960..d4ece4eff 100644 --- a/src/app/v1/_lib/proxy/discovery-validity.ts +++ b/src/app/v1/_lib/proxy/discovery-validity.ts @@ -137,11 +137,58 @@ export class DiscoveryValidityParser { push(chunk: Uint8Array | string): DiscoveryValidity { this.buffered += typeof chunk === "string" ? chunk : this.decoder.decode(chunk, { stream: true }); - const result = classifyDiscoveryChunk(this.buffered, this.protocol); + + // SSE streams are line framed. Consume each completed line once instead + // of reparsing the complete prefix on every chunk (which is quadratic on + // long streams). Keep only the unfinished line for the next push. + if (this.buffered.includes("\n")) { + const lines = this.buffered.split(/\r?\n/); + this.buffered = lines.pop() ?? ""; + for (const line of lines) this.consumeLine(line); + } + + // Some providers return one raw JSON object without an SSE newline. Parse + // it only when the complete object is available; incomplete JSON remains + // buffered and is not repeatedly scanned as a protocol event. + const tail = this.buffered.trim(); + if (tail) { + const candidate = tail.startsWith("data:") ? tail.slice(5).trim() : tail; + if (candidate === "[DONE]") { + this._terminal = true; + this.buffered = ""; + } else if (candidate.startsWith("{") || candidate.startsWith("[")) { + try { + const value = JSON.parse(candidate) as unknown; + this.consumeValue(value); + this.buffered = ""; + } catch { + // Keep incomplete raw JSON until the next chunk completes it. + } + } + } + + return { ready: this._ready && !this._error, terminal: this._terminal, error: this._error }; + } + + private consumeLine(line: string): void { + const candidate = line.startsWith("data:") ? line.slice(5).trim() : line.trim(); + if (!candidate || candidate.startsWith(":")) return; + if (candidate === "[DONE]") { + this._terminal = true; + return; + } + try { + this.consumeValue(JSON.parse(candidate) as unknown); + } catch { + // Ignore comments and incomplete/non-JSON protocol lines. + } + } + + private consumeValue(value: unknown): void { + const result = classifyJson(value, this.protocol); this._ready ||= result.ready; this._terminal ||= result.terminal; this._error ||= result.error; - return { ready: this._ready && !this._error, terminal: this._terminal, error: this._error }; } get ready(): boolean { diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 122e00d71..18182a2e1 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -63,7 +63,7 @@ import { buildProxyUrl } from "../url"; import { rectifyBillingHeader } from "./billing-header-rectifier"; import { bindClientAbortListener } from "./client-abort-listener"; import { deriveClientSafeUpstreamErrorMessage } from "./client-error-message"; -import { DiscoveryCoordinator } from "./discovery-coordinator"; +import { type DiscoveryAction, DiscoveryCoordinator } from "./discovery-coordinator"; import { type DiscoveryProtocol, DiscoveryValidityParser } from "./discovery-validity"; import { isStandardProxyEndpointPath } from "./endpoint-family-catalog"; import { resolveEndpointPolicy, shouldEnforceStrictEndpointPoolPolicy } from "./endpoint-policy"; @@ -4875,6 +4875,7 @@ export class ProxyForwarder { pending: boolean; ready: boolean; round: number; + readerTransferred: boolean; } >(); const launched = new Set(); @@ -4889,6 +4890,7 @@ export class ProxyForwarder { let totalTimer: NodeJS.Timeout | null = null; let roundTimer: NodeJS.Timeout | null = null; let stickyTimer: NodeJS.Timeout | null = null; + let executeCoordinatorAction: (action: DiscoveryAction) => Promise = async () => {}; let resolveResult: ((result: { response?: Response; error?: Error }) => void) | null = null; const resultPromise = new Promise<{ response?: Response; error?: Error }>((resolve) => { resolveResult = resolve; @@ -4904,6 +4906,7 @@ export class ProxyForwarder { }; const cancelAttempt = (attempt: (typeof winner & { id: string }) | null, reason: string) => { + if (attempt?.readerTransferred) return; if (!attempt?.pending) return; attempt.pending = false; try { @@ -4911,7 +4914,12 @@ export class ProxyForwarder { } catch { /* abort is best effort */ } - void attempt.reader?.cancel(reason).catch(() => undefined); + try { + const cancelPromise = attempt.reader?.cancel(reason); + cancelPromise?.catch(() => undefined); + } catch (error) { + logger.debug("[Discovery] Reader cancel failed", { reason, error }); + } try { attempt.releaseAgent?.(); } catch { @@ -4933,8 +4941,18 @@ export class ProxyForwarder { if (roundTimer) clearTimeout(roundTimer); if (stickyTimer) clearTimeout(stickyTimer); cancelLosers(); - const attempted = new Set(launched); - await ProxyForwarder.clearSessionProviderBindings(session, attempted); + if (bindingSnapshot) { + if (bindingSnapshot.providerId != null) { + await SessionManager.clearVersionedSessionProvider( + bindingSnapshot, + bindingSnapshot.providerId, + 0 + ); + } + } else { + const attempted = new Set(launched); + await ProxyForwarder.clearSessionProviderBindings(session, attempted); + } resolveResult?.({ error }); }; @@ -4943,6 +4961,9 @@ export class ProxyForwarder { committed = true; winner = attempt; attempt.pending = false; + // From this point ResponseHandler owns the reader and agent release. + // No coordinator/timer path may cancel or release this attempt again. + attempt.readerTransferred = true; if (totalTimer) clearTimeout(totalTimer); if (roundTimer) clearTimeout(roundTimer); if (stickyTimer) clearTimeout(stickyTimer); @@ -5044,6 +5065,7 @@ export class ProxyForwarder { pending: true, ready: false, round: currentRound, + readerTransferred: false, provider, session: attemptSession, baseUrl: endpoint.baseUrl, @@ -5080,6 +5102,7 @@ export class ProxyForwarder { pending: boolean; ready: boolean; round: number; + readerTransferred: boolean; }; attempts.set(id, attempt); coordinator.addAttempt({ @@ -5138,7 +5161,7 @@ export class ProxyForwarder { .catch(async (error) => { if (committed || settled || !attempt.pending) return; attempt.pending = false; - coordinator.markFailed(id); + const failureAction = coordinator.markFailed(id); lastError = error instanceof Error ? error : new Error(String(error)); lastErrorCategory = await categorizeErrorAsync(lastError); session.addProviderToChain(provider, { @@ -5156,8 +5179,17 @@ export class ProxyForwarder { } attempt.releaseAgent?.(); releaseProviderRef(provider.id); - const replacement = await chooseCandidate(); - if (replacement && !committed && !settled) await launch(replacement, "normal"); + const actionOwnsNextStep = + failureAction.type === "promote_fallback" || + failureAction.type === "launch" || + failureAction.type === "terminal_failure"; + if (actionOwnsNextStep) { + await executeCoordinatorAction(failureAction); + } + if (!actionOwnsNextStep && !committed && !settled) { + const replacement = await chooseCandidate(); + if (replacement) await launch(replacement, "normal"); + } if ( Array.from(attempts.values()).every((candidate) => !candidate.pending) && noMoreCandidates @@ -5166,6 +5198,12 @@ export class ProxyForwarder { ProxyForwarder.resolveHedgeTerminalError(lastError, lastErrorCategory) ); } + }) + .catch((error) => { + logger.warn("[Discovery] Attempt completion handler failed", { + providerId: provider.id, + error, + }); }); }; @@ -5184,60 +5222,75 @@ export class ProxyForwarder { } } if (!committed && !settled) { - roundTimer = setTimeout(() => void onBoundary(), discoverySlaMs); + roundTimer = setTimeout(() => { + void onBoundary().catch((error) => + logger.warn("[Discovery] Round boundary failed", { error }) + ); + }, discoverySlaMs); } }; - const onBoundary = async () => { + executeCoordinatorAction = async (action) => { if (settled || committed) return; - const fallback = Array.from(attempts.values()).find( - (attempt) => attempt.pending && attempt.kind === "fallback" - ); - const pendingNormals = Array.from(attempts.values()) - .filter((attempt) => attempt.pending && attempt.kind === "normal") - .sort( - (a, b) => - (a.provider.priority || 0) - (b.provider.priority || 0) || a.sequence - b.sequence - ); - const readyNormal = pendingNormals.find((attempt) => attempt.ready); - if (readyNormal) { - await commit(readyNormal); + if (action.type === "cancel" || action.type === "launch") { + const cancelIds = + action.type === "cancel" ? action.attemptIds : (action.cancelAttemptIds ?? []); + for (const id of cancelIds) { + const attempt = attempts.get(id); + // Coordinator marks cancelled attempts non-pending before returning + // the action. Restore the transport-facing state long enough for the + // exactly-once cancellation/release path to run. + if (attempt && !attempt.readerTransferred) attempt.pending = true; + if (attempt) cancelAttempt(attempt, "discovery_round_boundary"); + } + if (action.promoteAttemptId) { + const fallback = attempts.get(action.promoteAttemptId); + if (fallback) fallback.kind = "fallback"; + if (action.type === "cancel" && currentRound < maxRounds) { + await launchNextRound(); + } + } + } + if (action.type === "commit_normal" || action.type === "promote_fallback") { + const attempt = attempts.get(action.attemptId); + if (attempt) await commit(attempt); return; } - // The fallback is held during the SLA window, but at the round - // boundary it may take over when no normal result is ready. - if (fallback?.ready) { - await commit(fallback); + if (action.type === "launch") { + await launchNextRound(); return; } - if (pendingNormals[0]) { - if (fallback) { - for (const loser of pendingNormals) cancelAttempt(loser, "discovery_round_boundary"); - if (currentRound < maxRounds) await launchNextRound(); - return; - } - pendingNormals[0].kind = "fallback"; - for (const loser of pendingNormals.slice(1)) - cancelAttempt(loser, "discovery_round_boundary"); - if (currentRound < maxRounds) await launchNextRound(); + if (action.type === "terminal_failure") { + await settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); return; } - if (currentRound < maxRounds) await launchNextRound(); - else await settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); + if (action.type === "none") { + const fallbackPending = Array.from(attempts.values()).some( + (attempt) => attempt.pending && attempt.kind === "fallback" + ); + if (fallbackPending && currentRound < maxRounds) { + await launchNextRound(); + } + } + }; + + const onBoundary = async () => { + if (settled || committed) return; + await executeCoordinatorAction(coordinator.onRoundBoundary()); }; const cleanupAbort = bindClientAbortListener(session.clientAbortSignal, () => { if (settled || committed) return; - void settleFailure(new ProxyError("Request aborted by client", 499, undefined, true)); + void settleFailure(new ProxyError("Request aborted by client", 499, undefined, true)).catch( + (error) => logger.warn("[Discovery] Client abort cleanup failed", { error }) + ); }); totalTimer = setTimeout(() => { if (settled || committed) return; - const fallback = Array.from(attempts.values()).find( - (attempt) => attempt.pending && attempt.kind === "fallback" && attempt.ready + void executeCoordinatorAction(coordinator.onDeadline()).catch((error) => + logger.warn("[Discovery] Deadline action failed", { error }) ); - if (fallback) void commit(fallback); - else void settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); }, totalTimeoutMs); try { @@ -5267,8 +5320,15 @@ export class ProxyForwarder { logger.debug("[Discovery] Failed to clear timed-out Sticky", { error }) ); } - if (currentRound < maxRounds) void launchNextRound(); - else void onBoundary(); + if (currentRound < maxRounds) { + void launchNextRound().catch((error) => + logger.warn("[Discovery] Sticky round launch failed", { error }) + ); + } else { + void onBoundary().catch((error) => + logger.warn("[Discovery] Sticky boundary failed", { error }) + ); + } } }, stickySlaMs); } else { @@ -5285,7 +5345,11 @@ export class ProxyForwarder { noMoreCandidates = true; } } - roundTimer = setTimeout(() => void onBoundary(), discoverySlaMs); + roundTimer = setTimeout(() => { + void onBoundary().catch((error) => + logger.warn("[Discovery] Round boundary failed", { error }) + ); + }, discoverySlaMs); } const result = await resultPromise; if (result.error) throw result.error; diff --git a/tests/unit/proxy/discovery-coordinator.test.ts b/tests/unit/proxy/discovery-coordinator.test.ts index 70b09b6d0..35585c473 100644 --- a/tests/unit/proxy/discovery-coordinator.test.ts +++ b/tests/unit/proxy/discovery-coordinator.test.ts @@ -27,7 +27,11 @@ describe("DiscoveryCoordinator", () => { coordinator.addAttempt(attempt("a", 1)); coordinator.addAttempt(attempt("b", 2)); const action = coordinator.onRoundBoundary(); - expect(action.type).toBe("cancel"); + expect(action).toMatchObject({ + type: "launch", + promoteAttemptId: "a", + cancelAttemptIds: ["b"], + }); expect(coordinator.snapshot.find((item) => item.id === "a")?.kind).toBe("fallback"); expect(coordinator.snapshot.filter((item) => item.pending)).toHaveLength(1); }); @@ -45,9 +49,7 @@ describe("DiscoveryCoordinator", () => { it("promotes a ready fallback at the round boundary when no normal is ready", () => { const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); coordinator.addAttempt(attempt("a", 1, "fallback")); - coordinator.addAttempt(attempt("b", 1, "normal")); - expect(coordinator.markReady("a")).toEqual({ type: "none" }); - expect(coordinator.onRoundBoundary()).toEqual({ type: "promote_fallback", attemptId: "a" }); + expect(coordinator.markReady("a")).toEqual({ type: "promote_fallback", attemptId: "a" }); expect(coordinator.snapshot.find((item) => item.id === "a")?.kind).toBe("fallback"); }); @@ -60,4 +62,16 @@ describe("DiscoveryCoordinator", () => { coordinator.markReady("c"); expect(coordinator.onRoundBoundary()).toEqual({ type: "commit_normal", attemptId: "c" }); }); + + it("reports normal attempts cancelled when retaining an existing fallback", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 3 }); + coordinator.addAttempt(attempt("fallback", 1, "fallback")); + coordinator.addAttempt(attempt("normal", 2)); + const action = coordinator.onRoundBoundary(); + expect(action).toEqual({ + type: "launch", + slots: 1, + cancelAttemptIds: ["normal"], + }); + }); }); diff --git a/tests/unit/proxy/discovery-validity.test.ts b/tests/unit/proxy/discovery-validity.test.ts index 4d5e6cae2..83feb4f8e 100644 --- a/tests/unit/proxy/discovery-validity.test.ts +++ b/tests/unit/proxy/discovery-validity.test.ts @@ -60,4 +60,18 @@ describe("discovery validity", () => { ).ready ).toBe(true); }); + + it("consumes split SSE lines incrementally without waiting for the full stream", () => { + const parser = new DiscoveryValidityParser("openai-chat"); + expect(parser.push('data: {"choices":[{"delta":{"content":"hel')).toEqual({ + ready: false, + terminal: false, + error: false, + }); + expect(parser.push('lo"}}]}\n\n')).toEqual({ + ready: true, + terminal: false, + error: false, + }); + }); }); From 9c7793c86e29b6fc8fc4d56163d3c9c3e23e3d76 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 14:56:20 -0400 Subject: [PATCH 11/89] fix(proxy): guard optional discovery request message --- src/app/v1/_lib/proxy/forwarder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 18182a2e1..753e7d77f 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -3848,7 +3848,7 @@ export class ProxyForwarder { settings.discoveryEnabled === true && endpointPolicy.allowRetry && endpointPolicy.allowProviderSwitch && - message.stream === true && + message?.stream === true && !endpointPolicy.bypassForwarderPreprocessing && protocol !== "unknown" && routing.routingMode !== "lease_conflict_single" && From 3edd52b09721f9f579de313ca8e149465864b8c4 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 14:56:20 -0400 Subject: [PATCH 12/89] fix(proxy): guard optional discovery request message --- src/app/v1/_lib/proxy/forwarder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 18182a2e1..753e7d77f 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -3848,7 +3848,7 @@ export class ProxyForwarder { settings.discoveryEnabled === true && endpointPolicy.allowRetry && endpointPolicy.allowProviderSwitch && - message.stream === true && + message?.stream === true && !endpointPolicy.bypassForwarderPreprocessing && protocol !== "unknown" && routing.routingMode !== "lease_conflict_single" && From ee5557ecaf36aaf0915c58d6626128935de023ee Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 14:58:09 -0400 Subject: [PATCH 13/89] fix(proxy): handle replacement launch failures --- src/app/v1/_lib/proxy/forwarder.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 753e7d77f..a96f93e90 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -5300,8 +5300,19 @@ export class ProxyForwarder { } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); const replacement = await chooseCandidate(); - if (replacement) await launch(replacement, "normal"); - else await settleFailure(ProxyForwarder.resolveHedgeTerminalError(lastError, null)); + if (replacement) { + try { + await launch(replacement, "normal"); + } catch (replacementError) { + lastError = + replacementError instanceof Error + ? replacementError + : new Error(String(replacementError)); + await settleFailure(ProxyForwarder.resolveHedgeTerminalError(lastError, null)); + } + } else { + await settleFailure(ProxyForwarder.resolveHedgeTerminalError(lastError, null)); + } } const initial = hasSticky ? concurrency - 1 : Math.max(0, concurrency - 1); if (hasSticky) { From fcaf248cc1d1dadc19a8fdf0dc9d3f0713ed3130 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 14:58:09 -0400 Subject: [PATCH 14/89] fix(proxy): handle replacement launch failures --- src/app/v1/_lib/proxy/forwarder.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 753e7d77f..a96f93e90 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -5300,8 +5300,19 @@ export class ProxyForwarder { } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); const replacement = await chooseCandidate(); - if (replacement) await launch(replacement, "normal"); - else await settleFailure(ProxyForwarder.resolveHedgeTerminalError(lastError, null)); + if (replacement) { + try { + await launch(replacement, "normal"); + } catch (replacementError) { + lastError = + replacementError instanceof Error + ? replacementError + : new Error(String(replacementError)); + await settleFailure(ProxyForwarder.resolveHedgeTerminalError(lastError, null)); + } + } else { + await settleFailure(ProxyForwarder.resolveHedgeTerminalError(lastError, null)); + } } const initial = hasSticky ? concurrency - 1 : Math.max(0, concurrency - 1); if (hasSticky) { From 00de8abd8ddf907f66df1b033db74414e469fb5d Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 15:30:00 -0400 Subject: [PATCH 15/89] fix(discovery): close coordinator and configuration lifecycle gaps --- messages/zh-TW/settings/config.json | 4 +- src/actions/system-config.ts | 25 ++- .../_components/system-settings-form.tsx | 53 +++-- src/app/api/admin/system-config/route.ts | 2 +- .../v1/_lib/proxy/discovery-coordinator.ts | 34 +-- src/app/v1/_lib/proxy/forwarder.ts | 196 ++++++++++++------ src/lib/api-client/v1/openapi-types.gen.ts | 6 +- src/lib/api/v1/schemas/system-config.ts | 2 +- .../unit/proxy/discovery-coordinator.test.ts | 43 ++++ 9 files changed, 264 insertions(+), 101 deletions(-) diff --git a/messages/zh-TW/settings/config.json b/messages/zh-TW/settings/config.json index e510dd0c1..3e4a4b2dc 100644 --- a/messages/zh-TW/settings/config.json +++ b/messages/zh-TW/settings/config.json @@ -124,8 +124,8 @@ "discoveryEnabledDesc": "啟用後，冷啟動串流請求會在限定視窗內探測多個供應商，並且最多保留一個保底請求。預設關閉。", "discoveryConcurrency": "Discovery 首輪並發數", "maxDiscoveryRounds": "Discovery 最大輪數", - "discoverySlaMs": "探索 SLA（毫秒）", - "stickySlaMs": "Sticky 黏性 SLA（毫秒）", + "discoverySlaMs": "Discovery SLA（毫秒）", + "stickySlaMs": "Sticky SLA（毫秒）", "racingTotalTimeoutMs": "Discovery 總逾時（毫秒）", "stickyTimeoutCooldownMs": "Sticky 逾時冷卻（毫秒）", "discoveryWindowDesc": "總逾時必須不小於 Sticky SLA + 最大輪數 × Discovery SLA。Discovery 輸家會取消，不走舊 Hedge 的 drain 或輸家計費。", diff --git a/src/actions/system-config.ts b/src/actions/system-config.ts index 39464d90b..2bbf59be8 100644 --- a/src/actions/system-config.ts +++ b/src/actions/system-config.ts @@ -25,6 +25,13 @@ import type { } from "@/types/system-config"; import type { ActionResult } from "./types"; +const DEFAULT_DISCOVERY_WINDOW = { + discoverySlaMs: 10_000, + stickySlaMs: 20_000, + maxDiscoveryRounds: 2, + racingTotalTimeoutMs: 60_000, +} as const; + export async function fetchSystemSettings(): Promise> { try { const session = await getSession(); @@ -118,10 +125,20 @@ export async function saveSystemSettings(formData: { before = await getSystemSettings(); const validated = UpdateSystemSettingsSchema.parse(formData); const effectiveDiscoveryWindow = { - discoverySlaMs: validated.discoverySlaMs ?? before.discoverySlaMs, - stickySlaMs: validated.stickySlaMs ?? before.stickySlaMs, - maxDiscoveryRounds: validated.maxDiscoveryRounds ?? before.maxDiscoveryRounds, - racingTotalTimeoutMs: validated.racingTotalTimeoutMs ?? before.racingTotalTimeoutMs, + discoverySlaMs: + validated.discoverySlaMs ?? + before?.discoverySlaMs ?? + DEFAULT_DISCOVERY_WINDOW.discoverySlaMs, + stickySlaMs: + validated.stickySlaMs ?? before?.stickySlaMs ?? DEFAULT_DISCOVERY_WINDOW.stickySlaMs, + maxDiscoveryRounds: + validated.maxDiscoveryRounds ?? + before?.maxDiscoveryRounds ?? + DEFAULT_DISCOVERY_WINDOW.maxDiscoveryRounds, + racingTotalTimeoutMs: + validated.racingTotalTimeoutMs ?? + before?.racingTotalTimeoutMs ?? + DEFAULT_DISCOVERY_WINDOW.racingTotalTimeoutMs, }; if ( effectiveDiscoveryWindow.racingTotalTimeoutMs < diff --git a/src/app/[locale]/settings/config/_components/system-settings-form.tsx b/src/app/[locale]/settings/config/_components/system-settings-form.tsx index 26a4b7858..ae3307d22 100644 --- a/src/app/[locale]/settings/config/_components/system-settings-form.tsx +++ b/src/app/[locale]/settings/config/_components/system-settings-form.tsx @@ -114,6 +114,7 @@ function formatIpExtractionConfig(config: IpExtractionConfig): string { } const DEFAULT_IP_EXTRACTION_CONFIG_TEXT = formatIpExtractionConfig(DEFAULT_IP_EXTRACTION_CONFIG); +type DiscoveryNumberValue = number | ""; export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) { const router = useRouter(); @@ -138,16 +139,20 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) ); const [billHedgeLosers, setBillHedgeLosers] = useState(initialSettings.billHedgeLosers); const [discoveryEnabled, setDiscoveryEnabled] = useState(initialSettings.discoveryEnabled); - const [discoveryConcurrency, setDiscoveryConcurrency] = useState( + const [discoveryConcurrency, setDiscoveryConcurrency] = useState( initialSettings.discoveryConcurrency ); - const [maxDiscoveryRounds, setMaxDiscoveryRounds] = useState(initialSettings.maxDiscoveryRounds); - const [discoverySlaMs, setDiscoverySlaMs] = useState(initialSettings.discoverySlaMs); - const [stickySlaMs, setStickySlaMs] = useState(initialSettings.stickySlaMs); - const [racingTotalTimeoutMs, setRacingTotalTimeoutMs] = useState( + const [maxDiscoveryRounds, setMaxDiscoveryRounds] = useState( + initialSettings.maxDiscoveryRounds + ); + const [discoverySlaMs, setDiscoverySlaMs] = useState( + initialSettings.discoverySlaMs + ); + const [stickySlaMs, setStickySlaMs] = useState(initialSettings.stickySlaMs); + const [racingTotalTimeoutMs, setRacingTotalTimeoutMs] = useState( initialSettings.racingTotalTimeoutMs ); - const [stickyTimeoutCooldownMs, setStickyTimeoutCooldownMs] = useState( + const [stickyTimeoutCooldownMs, setStickyTimeoutCooldownMs] = useState( initialSettings.stickyTimeoutCooldownMs ); const [timezone, setTimezone] = useState(initialSettings.timezone); @@ -250,7 +255,27 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) return; } - if (racingTotalTimeoutMs < stickySlaMs + maxDiscoveryRounds * discoverySlaMs) { + const discoveryConfig = { + discoveryConcurrency: Number(discoveryConcurrency), + maxDiscoveryRounds: Number(maxDiscoveryRounds), + discoverySlaMs: Number(discoverySlaMs), + stickySlaMs: Number(stickySlaMs), + racingTotalTimeoutMs: Number(racingTotalTimeoutMs), + stickyTimeoutCooldownMs: Number(stickyTimeoutCooldownMs), + }; + if ( + discoveryEnabled && + Object.values(discoveryConfig).some((value) => !Number.isSafeInteger(value) || value < 1) + ) { + toast.error(t("discoveryWindowInvalid")); + return; + } + if ( + discoveryEnabled && + discoveryConfig.racingTotalTimeoutMs < + discoveryConfig.stickySlaMs + + discoveryConfig.maxDiscoveryRounds * discoveryConfig.discoverySlaMs + ) { toast.error(t("discoveryWindowInvalid")); return; } @@ -337,12 +362,7 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) billNonSuccessfulRequests, billHedgeLosers, discoveryEnabled, - discoveryConcurrency, - maxDiscoveryRounds, - discoverySlaMs, - stickySlaMs, - racingTotalTimeoutMs, - stickyTimeoutCooldownMs, + ...(discoveryEnabled ? discoveryConfig : {}), timezone, verboseProviderError, passThroughUpstreamErrorMessage, @@ -714,8 +734,11 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) id={`discovery-${key}`} type="number" min={min} - value={value} - onChange={(event) => setter(Number(event.target.value))} + required={discoveryEnabled} + value={value === 0 ? "" : value} + onChange={(event) => + setter(event.target.value === "" ? "" : Number(event.target.value)) + } disabled={isPending || !discoveryEnabled} className={inputClassName} /> diff --git a/src/app/api/admin/system-config/route.ts b/src/app/api/admin/system-config/route.ts index a5a925e73..06a7f091d 100644 --- a/src/app/api/admin/system-config/route.ts +++ b/src/app/api/admin/system-config/route.ts @@ -66,7 +66,7 @@ export async function POST(req: Request) { const racingTotalTimeoutMs = validated.racingTotalTimeoutMs ?? current.racingTotalTimeoutMs; if (racingTotalTimeoutMs < stickySlaMs + maxDiscoveryRounds * discoverySlaMs) { return Response.json( - { error: "竞速总超时必须不小于 Sticky SLA + Discovery 轮数 × Discovery SLA" }, + { error: "discoveryWindowInvalid", errorCode: "discoveryWindowInvalid" }, { status: 400 } ); } diff --git a/src/app/v1/_lib/proxy/discovery-coordinator.ts b/src/app/v1/_lib/proxy/discovery-coordinator.ts index 4d4462b2c..92bb0880c 100644 --- a/src/app/v1/_lib/proxy/discovery-coordinator.ts +++ b/src/app/v1/_lib/proxy/discovery-coordinator.ts @@ -67,6 +67,10 @@ export class DiscoveryCoordinator { } beginRound(): { requestEpoch: number; roundEpoch: number; round: number } { + if (this.isTerminal || this.round >= this.maxRounds) { + return { ...this.epochs, round: this.round }; + } + this.round += 1; this.roundEpoch += 1; if (!this.isTerminal) this.state = "DISCOVERY_RACING"; return { ...this.epochs, round: this.round }; @@ -83,7 +87,11 @@ export class DiscoveryCoordinator { } get isTerminal(): boolean { - return this.state === "WINNER_COMMITTED" || this.state === "TERMINAL_FAILED"; + return ( + this.state === "WINNER_COMMITTED" || + this.state === "FALLBACK_ACTIVE" || + this.state === "TERMINAL_FAILED" + ); } get activeAttempts(): DiscoveryAttempt[] { @@ -132,6 +140,8 @@ export class DiscoveryCoordinator { if (!attempt) return { type: "none" }; attempt.pending = false; attempt.ready = false; + const readyAction = this.chooseReadyNormal(); + if (readyAction.type !== "none") return readyAction; return this.afterAttemptState(); } @@ -152,8 +162,7 @@ export class DiscoveryCoordinator { ); if (higherTierPending) return { type: "none" }; } - const sameTier = readyNormal.filter((attempt) => attempt.priority === bestPriority); - const winner = sameTier[0]; + const winner = readyNormal[0]; this.state = "WINNER_COMMITTED"; winner.pending = false; return { @@ -188,12 +197,11 @@ export class DiscoveryCoordinator { const cancelAttemptIds = pendingNormal.map((attempt) => attempt.id); for (const attempt of pendingNormal) attempt.pending = false; if (this.round < this.maxRounds) { - this.round += 1; - this.roundEpoch += 1; + this.beginRound(); this.state = "DISCOVERY_RACING"; return { type: "launch", - slots: Math.max(1, this.concurrency - 1), + slots: Math.max(0, this.concurrency - 1), cancelAttemptIds, }; } @@ -214,12 +222,11 @@ export class DiscoveryCoordinator { for (const id of losers) this.attempts.get(id)!.pending = false; if (this.round < this.maxRounds) { - this.round += 1; - this.roundEpoch += 1; + this.beginRound(); this.state = "DISCOVERY_RACING"; return { type: "launch", - slots: Math.max(1, this.concurrency - 1), + slots: Math.max(0, this.concurrency - 1), cancelAttemptIds: losers, promoteAttemptId: fallback.id, }; @@ -229,6 +236,8 @@ export class DiscoveryCoordinator { onDeadline(): DiscoveryAction { if (this.isTerminal) return { type: "none" }; + const readyNormal = this.chooseReadyNormal(true); + if (readyNormal.type === "commit_normal") return readyNormal; const fallback = Array.from(this.attempts.values()).find( (attempt) => attempt.pending && attempt.kind === "fallback" && attempt.ready ); @@ -245,7 +254,7 @@ export class DiscoveryCoordinator { const attempt = this.attempts.get(id); if (!attempt || this.isTerminal) return { type: "none" }; attempt.pending = false; - this.state = "WINNER_COMMITTED"; + this.state = attempt.kind === "fallback" ? "FALLBACK_ACTIVE" : "WINNER_COMMITTED"; return { type: attempt.kind === "fallback" ? "promote_fallback" : "commit_normal", attemptId: id, @@ -276,9 +285,8 @@ export class DiscoveryCoordinator { this.state = "TERMINAL_FAILED"; return { type: "terminal_failure" }; } - this.round += 1; - this.roundEpoch += 1; + this.beginRound(); this.state = "DISCOVERY_RACING"; - return { type: "launch", slots: Math.max(1, this.concurrency - 1) }; + return { type: "launch", slots: this.concurrency }; } } diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index a96f93e90..a90a6c622 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -3845,7 +3845,6 @@ export class ProxyForwarder { disableStreamingHedge?: boolean; }; return ( - settings.discoveryEnabled === true && endpointPolicy.allowRetry && endpointPolicy.allowProviderSwitch && message?.stream === true && @@ -5035,6 +5034,14 @@ export class ProxyForwarder { const launch = async (provider: Provider, kind: "normal" | "fallback"): Promise => { if (settled || committed || launched.has(provider.id)) return; launched.add(provider.id); + let providerSessionRefTracked = false; + const rollbackLaunch = () => { + launched.delete(provider.id); + if (providerSessionRefTracked) { + releaseProviderRef(provider.id); + providerSessionRefTracked = false; + } + }; if (provider.id !== initialProvider.id && session.sessionId) { const limit = provider.limitConcurrentSessions || 0; const check = await RateLimitService.checkAndTrackProviderSession( @@ -5046,14 +5053,29 @@ export class ProxyForwarder { launched.delete(provider.id); throw new ProxyError(check.reason || "Provider concurrent limit reached", 503); } - if (check.referenced) session.recordProviderSessionRef(provider.id); + if (check.referenced) { + session.recordProviderSessionRef(provider.id); + providerSessionRefTracked = true; + } + } + let endpoint: Awaited>; + try { + endpoint = await ProxyForwarder.resolveStreamingHedgeEndpoint(session, provider); + } catch (error) { + rollbackLaunch(); + throw error; + } + let attemptSession: ProxySession; + try { + attemptSession = + provider.id === initialProvider.id + ? session + : ProxyForwarder.createStreamingShadowSession(session, provider); + attemptSession.setProvider(provider); + } catch (error) { + rollbackLaunch(); + throw error; } - const endpoint = await ProxyForwarder.resolveStreamingHedgeEndpoint(session, provider); - const attemptSession = - provider.id === initialProvider.id - ? session - : ProxyForwarder.createStreamingShadowSession(session, provider); - attemptSession.setProvider(provider); const controller = new AbortController(); const id = `${provider.id}:${sequence + 1}`; const attempt = { @@ -5137,26 +5159,43 @@ export class ProxyForwarder { attempt.reader = response.body.getReader(); while (!committed && !settled && attempt.pending) { const item = await attempt.reader.read(); - if (item.done) throw new EmptyResponseError(provider.id, provider.name, "empty_body"); + if (attempt.readerTransferred || committed || settled || !attempt.pending) break; + if (item.done) { + if (attempt.ready) break; + throw new EmptyResponseError(provider.id, provider.name, "empty_body"); + } if (!item.value || item.value.byteLength === 0) continue; attempt.chunks.push(item.value); const validity = attempt.parser.push(item.value); - if (validity.error || validity.terminal) + if (validity.error) throw new ProxyError("Invalid upstream discovery response", 502); + if (validity.ready) attempt.ready = true; + if (validity.terminal) { + if (attempt.ready) break; throw new ProxyError("Invalid upstream discovery response", 502); + } if (!validity.ready) continue; - attempt.ready = true; const normalPendingHigher = Array.from(attempts.values()).some( (other) => other.pending && other.kind === "normal" && (other.provider.priority || 0) < (provider.priority || 0) ); - if (normalPendingHigher) continue; + // Once a valid prefix is buffered behind a higher-priority normal + // attempt, stop reading. This leaves the reader idle so a later + // fallback promotion can transfer it without racing an in-flight + // read and losing bytes from the buffered response. + if (normalPendingHigher) return; const action = coordinator.markReady(id); if (action.type === "commit_normal" || action.type === "promote_fallback") await commit(attempt); return; } + if (attempt.ready && !committed && !settled && attempt.pending) { + const action = coordinator.markReady(id); + if (action.type === "commit_normal" || action.type === "promote_fallback") { + await commit(attempt); + } + } }) .catch(async (error) => { if (committed || settled || !attempt.pending) return; @@ -5180,6 +5219,7 @@ export class ProxyForwarder { attempt.releaseAgent?.(); releaseProviderRef(provider.id); const actionOwnsNextStep = + failureAction.type === "commit_normal" || failureAction.type === "promote_fallback" || failureAction.type === "launch" || failureAction.type === "terminal_failure"; @@ -5188,7 +5228,15 @@ export class ProxyForwarder { } if (!actionOwnsNextStep && !committed && !settled) { const replacement = await chooseCandidate(); - if (replacement) await launch(replacement, "normal"); + if (replacement) { + try { + await launch(replacement, "normal"); + } catch (launchError) { + lastError = + launchError instanceof Error ? launchError : new Error(String(launchError)); + noMoreCandidates = true; + } + } } if ( Array.from(attempts.values()).every((candidate) => !candidate.pending) && @@ -5207,20 +5255,37 @@ export class ProxyForwarder { }); }; - const launchNextRound = async () => { + const launchNextRound = async (slots: number, coordinatorAlreadyAdvanced = false) => { if (settled || committed) return; - currentRound += 1; - if (currentRound > maxRounds) return; - coordinator.beginRound(); - const candidate = await chooseCandidate(); - if (candidate) { + if (coordinatorAlreadyAdvanced) { + currentRound = coordinator.round; + } else { + const nextRound = coordinator.beginRound(); + currentRound = nextRound.round; + } + if (currentRound > maxRounds || slots <= 0) return; + const candidates = await ProxyProviderResolver.pickDiscoveryProviders( + session, + slots, + Array.from(launched) + ); + if (candidates.length === 0) { + noMoreCandidates = true; + } + for (const candidate of candidates) { try { await launch(candidate, "normal"); } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); - noMoreCandidates = true; + noMoreCandidates = false; } } + const hasPendingAttempt = Array.from(attempts.values()).some((attempt) => attempt.pending); + if (!hasPendingAttempt) { + await settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); + return; + } + if (candidates.length === 0) return; if (!committed && !settled) { roundTimer = setTimeout(() => { void onBoundary().catch((error) => @@ -5246,8 +5311,9 @@ export class ProxyForwarder { if (action.promoteAttemptId) { const fallback = attempts.get(action.promoteAttemptId); if (fallback) fallback.kind = "fallback"; - if (action.type === "cancel" && currentRound < maxRounds) { - await launchNextRound(); + if (action.type === "cancel") { + if (fallback) await commit(fallback); + return; } } } @@ -5257,21 +5323,13 @@ export class ProxyForwarder { return; } if (action.type === "launch") { - await launchNextRound(); + await launchNextRound(action.slots, true); return; } if (action.type === "terminal_failure") { await settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); return; } - if (action.type === "none") { - const fallbackPending = Array.from(attempts.values()).some( - (attempt) => attempt.pending && attempt.kind === "fallback" - ); - if (fallbackPending && currentRound < maxRounds) { - await launchNextRound(); - } - } }; const onBoundary = async () => { @@ -5279,6 +5337,7 @@ export class ProxyForwarder { await executeCoordinatorAction(coordinator.onRoundBoundary()); }; + let stickyProbeFailed = false; const cleanupAbort = bindClientAbortListener(session.clientAbortSignal, () => { if (settled || committed) return; void settleFailure(new ProxyError("Request aborted by client", 499, undefined, true)).catch( @@ -5298,6 +5357,7 @@ export class ProxyForwarder { try { await launch(initialProvider, "normal"); } catch (error) { + stickyProbeFailed = hasSticky; lastError = error instanceof Error ? error : new Error(String(error)); const replacement = await chooseCandidate(); if (replacement) { @@ -5316,32 +5376,40 @@ export class ProxyForwarder { } const initial = hasSticky ? concurrency - 1 : Math.max(0, concurrency - 1); if (hasSticky) { - stickyTimer = setTimeout(() => { - const sticky = Array.from(attempts.values()).find( - (attempt) => attempt.pending && attempt.provider.id === initialProvider.id - ); - if (sticky) { - sticky.kind = "fallback"; - if (bindingSnapshot && bindingSnapshot.providerId === initialProvider.id) { - void SessionManager.clearVersionedSessionProvider( - bindingSnapshot, - initialProvider.id, - Math.ceil((settings.stickyTimeoutCooldownMs ?? 300_000) / 1000) - ).catch((error) => - logger.debug("[Discovery] Failed to clear timed-out Sticky", { error }) - ); - } - if (currentRound < maxRounds) { - void launchNextRound().catch((error) => - logger.warn("[Discovery] Sticky round launch failed", { error }) - ); - } else { - void onBoundary().catch((error) => - logger.warn("[Discovery] Sticky boundary failed", { error }) - ); + if (stickyProbeFailed) { + roundTimer = setTimeout(() => { + void onBoundary().catch((error) => + logger.warn("[Discovery] Sticky failure boundary failed", { error }) + ); + }, discoverySlaMs); + } else { + stickyTimer = setTimeout(() => { + const sticky = Array.from(attempts.values()).find( + (attempt) => attempt.pending && attempt.provider.id === initialProvider.id + ); + if (sticky) { + sticky.kind = "fallback"; + if (bindingSnapshot && bindingSnapshot.providerId === initialProvider.id) { + void SessionManager.clearVersionedSessionProvider( + bindingSnapshot, + initialProvider.id, + Math.ceil((settings.stickyTimeoutCooldownMs ?? 300_000) / 1000) + ).catch((error) => + logger.debug("[Discovery] Failed to clear timed-out Sticky", { error }) + ); + } + if (currentRound < maxRounds) { + void launchNextRound(Math.max(0, concurrency - 1)).catch((error) => + logger.warn("[Discovery] Sticky round launch failed", { error }) + ); + } else { + void onBoundary().catch((error) => + logger.warn("[Discovery] Sticky boundary failed", { error }) + ); + } } - } - }, stickySlaMs); + }, stickySlaMs); + } } else { const candidates = await ProxyProviderResolver.pickDiscoveryProviders( session, @@ -5353,14 +5421,18 @@ export class ProxyForwarder { await launch(provider, "normal"); } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); - noMoreCandidates = true; + noMoreCandidates = false; } } - roundTimer = setTimeout(() => { - void onBoundary().catch((error) => - logger.warn("[Discovery] Round boundary failed", { error }) - ); - }, discoverySlaMs); + if (Array.from(attempts.values()).some((attempt) => attempt.pending)) { + roundTimer = setTimeout(() => { + void onBoundary().catch((error) => + logger.warn("[Discovery] Round boundary failed", { error }) + ); + }, discoverySlaMs); + } else if (!settled && !committed) { + await settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); + } } const result = await resultPromise; if (result.error) throw result.error; diff --git a/src/lib/api-client/v1/openapi-types.gen.ts b/src/lib/api-client/v1/openapi-types.gen.ts index dd0ab0996..ad00a4ecd 100644 --- a/src/lib/api-client/v1/openapi-types.gen.ts +++ b/src/lib/api-client/v1/openapi-types.gen.ts @@ -11891,7 +11891,7 @@ export interface operations { discoveryConcurrency: number; /** @description Maximum number of Discovery rounds. */ maxDiscoveryRounds: number; - /** @description 首字 Discovery SLA in milliseconds. */ + /** @description 首字节 Discovery SLA in milliseconds. */ discoverySlaMs: number; /** @description Sticky probe SLA in milliseconds. */ stickySlaMs: number; @@ -12167,7 +12167,7 @@ export interface operations { discoveryConcurrency?: number; /** @description Maximum number of Discovery rounds. */ maxDiscoveryRounds?: number; - /** @description 首字 Discovery SLA in milliseconds. */ + /** @description 首字节 Discovery SLA in milliseconds. */ discoverySlaMs?: number; /** @description Sticky probe SLA in milliseconds. */ stickySlaMs?: number; @@ -12316,7 +12316,7 @@ export interface operations { discoveryConcurrency: number; /** @description Maximum number of Discovery rounds. */ maxDiscoveryRounds: number; - /** @description 首字 Discovery SLA in milliseconds. */ + /** @description 首字节 Discovery SLA in milliseconds. */ discoverySlaMs: number; /** @description Sticky probe SLA in milliseconds. */ stickySlaMs: number; diff --git a/src/lib/api/v1/schemas/system-config.ts b/src/lib/api/v1/schemas/system-config.ts index e52e21b17..f27d5993e 100644 --- a/src/lib/api/v1/schemas/system-config.ts +++ b/src/lib/api/v1/schemas/system-config.ts @@ -105,7 +105,7 @@ export const SystemSettingsSchema = z .positive() .describe("Maximum number of normal Discovery attempts in the initial batch."), maxDiscoveryRounds: z.number().int().positive().describe("Maximum number of Discovery rounds."), - discoverySlaMs: z.number().int().positive().describe("首字 Discovery SLA in milliseconds."), + discoverySlaMs: z.number().int().positive().describe("首字节 Discovery SLA in milliseconds."), stickySlaMs: z.number().int().positive().describe("Sticky probe SLA in milliseconds."), racingTotalTimeoutMs: z .number() diff --git a/tests/unit/proxy/discovery-coordinator.test.ts b/tests/unit/proxy/discovery-coordinator.test.ts index 35585c473..9aeede802 100644 --- a/tests/unit/proxy/discovery-coordinator.test.ts +++ b/tests/unit/proxy/discovery-coordinator.test.ts @@ -74,4 +74,47 @@ describe("DiscoveryCoordinator", () => { cancelAttemptIds: ["normal"], }); }); + + it("commits a ready lower-priority candidate when the higher tier fails", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("high", 1)); + coordinator.addAttempt(attempt("low", 10)); + expect(coordinator.markReady("low")).toEqual({ type: "none" }); + expect(coordinator.markFailed("high")).toEqual({ + type: "commit_normal", + attemptId: "low", + }); + }); + + it("treats fallback promotion as terminal", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("fallback", 1, "fallback")); + + expect(coordinator.markReady("fallback")).toEqual({ + type: "promote_fallback", + attemptId: "fallback", + }); + expect(coordinator.state).toBe("FALLBACK_ACTIVE"); + expect(coordinator.isTerminal).toBe(true); + expect(coordinator.markFailed("fallback")).toEqual({ type: "none" }); + }); + + it("opens a full new round when all normal attempts fail", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 3, maxRounds: 2 }); + coordinator.addAttempt(attempt("a", 1)); + coordinator.addAttempt(attempt("b", 2)); + + expect(coordinator.markFailed("a")).toEqual({ type: "none" }); + expect(coordinator.markFailed("b")).toEqual({ type: "launch", slots: 3 }); + expect(coordinator.round).toBe(2); + }); + + it("commits a ready normal candidate at the total deadline", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("high", 1)); + coordinator.addAttempt(attempt("normal", 2)); + coordinator.markReady("normal"); + + expect(coordinator.onDeadline()).toEqual({ type: "commit_normal", attemptId: "normal" }); + }); }); From 420802af9873bc9f2d76180e0a0abd539ef05d29 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 15:34:36 -0400 Subject: [PATCH 16/89] docs(discovery): document Redis cluster capability fallback --- docs/streaming-discovery.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/streaming-discovery.md b/docs/streaming-discovery.md index 694d560d0..39d261241 100644 --- a/docs/streaming-discovery.md +++ b/docs/streaming-discovery.md @@ -46,6 +46,14 @@ the versioned Redis binding capability is available. If Redis capability is unknown/unavailable, the existing provider selection and Hedge behavior remain active. +The versioned binding scripts require the canonical binding, legacy provider, +legacy owner, lease, and cooldown keys to be evaluated atomically. On Redis +Cluster layouts where those keys do not share a slot and Redis returns +`CROSSSLOT` (or when Lua capability probing fails), the capability state is +`unavailable`; the service records that state and uses the tenant-checked +legacy wrapper. Discovery stays disabled until a later connection-lifecycle +probe succeeds. + ## Rollout 1. Apply the system-settings migration. From 96cbae653d4fbf9dc4928ad462a5f8a1ca69fc51 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 15:36:40 -0400 Subject: [PATCH 17/89] fix(discovery): synchronize Sticky fallback promotion --- src/app/v1/_lib/proxy/discovery-coordinator.ts | 10 ++++++++++ src/app/v1/_lib/proxy/forwarder.ts | 1 + tests/unit/proxy/discovery-coordinator.test.ts | 8 ++++++++ 3 files changed, 19 insertions(+) diff --git a/src/app/v1/_lib/proxy/discovery-coordinator.ts b/src/app/v1/_lib/proxy/discovery-coordinator.ts index 92bb0880c..e6937e7c0 100644 --- a/src/app/v1/_lib/proxy/discovery-coordinator.ts +++ b/src/app/v1/_lib/proxy/discovery-coordinator.ts @@ -86,6 +86,16 @@ export class DiscoveryCoordinator { this.attempts.delete(id); } + /** Mark an already-running attempt as the sole fallback for this request. */ + promoteToFallback(id: string): boolean { + if (this.isTerminal) return false; + const attempt = this.attempts.get(id); + if (!attempt?.pending) return false; + attempt.kind = "fallback"; + this.state = "FALLBACK_READY_HELD"; + return true; + } + get isTerminal(): boolean { return ( this.state === "WINNER_COMMITTED" || diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index a90a6c622..033d28c53 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -5389,6 +5389,7 @@ export class ProxyForwarder { ); if (sticky) { sticky.kind = "fallback"; + coordinator.promoteToFallback(sticky.id); if (bindingSnapshot && bindingSnapshot.providerId === initialProvider.id) { void SessionManager.clearVersionedSessionProvider( bindingSnapshot, diff --git a/tests/unit/proxy/discovery-coordinator.test.ts b/tests/unit/proxy/discovery-coordinator.test.ts index 9aeede802..69dee7127 100644 --- a/tests/unit/proxy/discovery-coordinator.test.ts +++ b/tests/unit/proxy/discovery-coordinator.test.ts @@ -99,6 +99,14 @@ describe("DiscoveryCoordinator", () => { expect(coordinator.markFailed("fallback")).toEqual({ type: "none" }); }); + it("keeps coordinator kind in sync when a running Sticky becomes fallback", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("sticky", 1)); + + expect(coordinator.promoteToFallback("sticky")).toBe(true); + expect(coordinator.snapshot.find((item) => item.id === "sticky")?.kind).toBe("fallback"); + }); + it("opens a full new round when all normal attempts fail", () => { const coordinator = new DiscoveryCoordinator({ concurrency: 3, maxRounds: 2 }); coordinator.addAttempt(attempt("a", 1)); From e453b8e78459bc6be8112dffc722b519eed78b68 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 15:38:27 -0400 Subject: [PATCH 18/89] fix(discovery): probe binding capability before eligibility --- src/app/v1/_lib/proxy/forwarder.ts | 11 +++++++---- src/lib/session-manager.ts | 5 +++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 033d28c53..1915831b1 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -3831,10 +3831,13 @@ export class ProxyForwarder { if (settings.discoveryEnabled !== true) { return false; } - if ( - typeof SessionManager.getVersionedBindingCapabilityState !== "function" || - SessionManager.getVersionedBindingCapabilityState() !== "available" - ) { + const capabilityState = + typeof SessionManager.ensureVersionedBindingCapability === "function" + ? await SessionManager.ensureVersionedBindingCapability() + : typeof SessionManager.getVersionedBindingCapabilityState === "function" + ? SessionManager.getVersionedBindingCapabilityState() + : "unknown"; + if (capabilityState !== "available") { return false; } const endpointPolicy = ProxyForwarder.getEndpointPolicy(session); diff --git a/src/lib/session-manager.ts b/src/lib/session-manager.ts index b47b9a3bf..f73c33fe0 100644 --- a/src/lib/session-manager.ts +++ b/src/lib/session-manager.ts @@ -34,6 +34,7 @@ import { import { clearSessionBinding as clearVersionedSessionBinding, compareAndSetSessionBinding, + ensureVersionedBindingCapability, mutateLegacySessionBindingSafely, readOrReconcileSessionBinding, isSessionProviderCoolingDown as readSessionProviderCooldown, @@ -668,6 +669,10 @@ export class SessionManager { return readVersionedBindingCapabilityState(); } + static async ensureVersionedBindingCapability(): Promise { + return ensureVersionedBindingCapability(); + } + static async getSessionBindingSnapshot( sessionId: string, keyId: number From 4c9690d46baf53ccd660b5720bb16bb954a6a5fb Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 15:42:46 -0400 Subject: [PATCH 19/89] fix(discovery): reject stale attempt boundary events --- .../v1/_lib/proxy/discovery-coordinator.ts | 2 +- src/app/v1/_lib/proxy/forwarder.ts | 27 +++++++++---------- .../unit/proxy/discovery-coordinator.test.ts | 1 + 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/app/v1/_lib/proxy/discovery-coordinator.ts b/src/app/v1/_lib/proxy/discovery-coordinator.ts index e6937e7c0..030ccd38c 100644 --- a/src/app/v1/_lib/proxy/discovery-coordinator.ts +++ b/src/app/v1/_lib/proxy/discovery-coordinator.ts @@ -147,7 +147,7 @@ export class DiscoveryCoordinator { ): DiscoveryAction { if (!this.acceptsEpoch(requestEpoch, roundEpoch) || this.isTerminal) return { type: "none" }; const attempt = this.attempts.get(id); - if (!attempt) return { type: "none" }; + if (!attempt?.pending) return { type: "none" }; attempt.pending = false; attempt.ready = false; const readyAction = this.chooseReadyNormal(); diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 1915831b1..59de0b133 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -5034,6 +5034,15 @@ export class ProxyForwarder { return candidates[0]; }; + const scheduleRoundBoundary = (delayMs: number) => { + const epoch = coordinator.epochs; + roundTimer = setTimeout(() => { + void executeCoordinatorAction( + coordinator.onRoundBoundary(epoch.requestEpoch, epoch.roundEpoch) + ).catch((error) => logger.warn("[Discovery] Round boundary failed", { error })); + }, delayMs); + }; + const launch = async (provider: Provider, kind: "normal" | "fallback"): Promise => { if (settled || committed || launched.has(provider.id)) return; launched.add(provider.id); @@ -5290,11 +5299,7 @@ export class ProxyForwarder { } if (candidates.length === 0) return; if (!committed && !settled) { - roundTimer = setTimeout(() => { - void onBoundary().catch((error) => - logger.warn("[Discovery] Round boundary failed", { error }) - ); - }, discoverySlaMs); + scheduleRoundBoundary(discoverySlaMs); } }; @@ -5380,11 +5385,7 @@ export class ProxyForwarder { const initial = hasSticky ? concurrency - 1 : Math.max(0, concurrency - 1); if (hasSticky) { if (stickyProbeFailed) { - roundTimer = setTimeout(() => { - void onBoundary().catch((error) => - logger.warn("[Discovery] Sticky failure boundary failed", { error }) - ); - }, discoverySlaMs); + scheduleRoundBoundary(discoverySlaMs); } else { stickyTimer = setTimeout(() => { const sticky = Array.from(attempts.values()).find( @@ -5429,11 +5430,7 @@ export class ProxyForwarder { } } if (Array.from(attempts.values()).some((attempt) => attempt.pending)) { - roundTimer = setTimeout(() => { - void onBoundary().catch((error) => - logger.warn("[Discovery] Round boundary failed", { error }) - ); - }, discoverySlaMs); + scheduleRoundBoundary(discoverySlaMs); } else if (!settled && !committed) { await settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); } diff --git a/tests/unit/proxy/discovery-coordinator.test.ts b/tests/unit/proxy/discovery-coordinator.test.ts index 69dee7127..69a7bbbeb 100644 --- a/tests/unit/proxy/discovery-coordinator.test.ts +++ b/tests/unit/proxy/discovery-coordinator.test.ts @@ -44,6 +44,7 @@ describe("DiscoveryCoordinator", () => { expect(coordinator.markReady("a", epoch.requestEpoch, epoch.roundEpoch)).toEqual({ type: "none", }); + expect(coordinator.markFailed("a")).toEqual({ type: "none" }); }); it("promotes a ready fallback at the round boundary when no normal is ready", () => { From 43c6733e434963e5500e9c33dfc7b504771977fb Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 15:55:23 -0400 Subject: [PATCH 20/89] fix(proxy): preserve complete discovery candidates --- .../v1/_lib/proxy/discovery-coordinator.ts | 7 +++++ src/app/v1/_lib/proxy/forwarder.ts | 26 ++++++++++++------- .../unit/proxy/discovery-coordinator.test.ts | 13 ++++++++++ tests/unit/proxy/discovery-validity.test.ts | 12 +++++++++ 4 files changed, 49 insertions(+), 9 deletions(-) diff --git a/src/app/v1/_lib/proxy/discovery-coordinator.ts b/src/app/v1/_lib/proxy/discovery-coordinator.ts index 4d4462b2c..a38d0813b 100644 --- a/src/app/v1/_lib/proxy/discovery-coordinator.ts +++ b/src/app/v1/_lib/proxy/discovery-coordinator.ts @@ -264,6 +264,13 @@ export class DiscoveryCoordinator { private afterAttemptState(): DiscoveryAction { const pending = this.activeAttempts; if (pending.length === 0) return this.finishOrLaunch(); + + // A higher-priority attempt may have been the only gate preventing a + // ready lower-priority candidate from winning. Once that attempt fails, + // re-run the normal winner selection before waiting for another boundary. + const readyNormal = this.chooseReadyNormal(); + if (readyNormal.type === "commit_normal") return readyNormal; + const fallback = pending.find((attempt) => attempt.kind === "fallback"); if (fallback?.ready && pending.every((attempt) => attempt.kind === "fallback")) { return this.commitWinner(fallback.id); diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index a96f93e90..7648319c5 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -5137,24 +5137,32 @@ export class ProxyForwarder { attempt.reader = response.body.getReader(); while (!committed && !settled && attempt.pending) { const item = await attempt.reader.read(); - if (item.done) throw new EmptyResponseError(provider.id, provider.name, "empty_body"); + if (item.done) { + // A ready candidate may have reached EOF while waiting for a + // higher-priority attempt. Its buffered prefix remains a valid + // response and must stay promotable. + if (attempt.ready) return; + throw new EmptyResponseError(provider.id, provider.name, "empty_body"); + } if (!item.value || item.value.byteLength === 0) continue; attempt.chunks.push(item.value); const validity = attempt.parser.push(item.value); - if (validity.error || validity.terminal) + // A single read can contain both deliverable content and the + // protocol terminator. Terminal is only invalid when no content + // was observed; otherwise the buffered candidate is complete. + if (validity.error || (validity.terminal && !validity.ready)) throw new ProxyError("Invalid upstream discovery response", 502); if (!validity.ready) continue; attempt.ready = true; - const normalPendingHigher = Array.from(attempts.values()).some( - (other) => - other.pending && - other.kind === "normal" && - (other.provider.priority || 0) < (provider.priority || 0) - ); - if (normalPendingHigher) continue; const action = coordinator.markReady(id); if (action.type === "commit_normal" || action.type === "promote_fallback") await commit(attempt); + // Do not issue another reader request after a complete candidate; + // the buffered stream is already sufficient for later promotion. + if (validity.terminal) return; + // The coordinator owns priority gating. A ready lower-priority + // candidate stays held while a higher tier is still pending. + if (action.type === "none") continue; return; } }) diff --git a/tests/unit/proxy/discovery-coordinator.test.ts b/tests/unit/proxy/discovery-coordinator.test.ts index 35585c473..328a02e97 100644 --- a/tests/unit/proxy/discovery-coordinator.test.ts +++ b/tests/unit/proxy/discovery-coordinator.test.ts @@ -74,4 +74,17 @@ describe("DiscoveryCoordinator", () => { cancelAttemptIds: ["normal"], }); }); + + it("retains a lower-priority ready candidate until the higher tier fails", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("high", 1)); + coordinator.addAttempt(attempt("low", 10)); + + expect(coordinator.markReady("low")).toEqual({ type: "none" }); + expect(coordinator.snapshot.find((item) => item.id === "low")).toMatchObject({ + ready: true, + pending: true, + }); + expect(coordinator.markFailed("high")).toEqual({ type: "commit_normal", attemptId: "low" }); + }); }); diff --git a/tests/unit/proxy/discovery-validity.test.ts b/tests/unit/proxy/discovery-validity.test.ts index 83feb4f8e..408fca35a 100644 --- a/tests/unit/proxy/discovery-validity.test.ts +++ b/tests/unit/proxy/discovery-validity.test.ts @@ -74,4 +74,16 @@ describe("discovery validity", () => { error: false, }); }); + + it("keeps ready when content and the terminal marker arrive in one read", () => { + const parser = new DiscoveryValidityParser("openai-chat"); + + expect( + parser.push('data: {"choices":[{"delta":{"content":"done"}}]}\n\ndata: [DONE]\n\n') + ).toEqual({ + ready: true, + terminal: true, + error: false, + }); + }); }); From 8cbf3be719195b4a7d90c64c2b1b38cd82a79c68 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 15:58:35 -0400 Subject: [PATCH 21/89] fix(discovery): honor configured sticky SLA and validation --- drizzle/0110_daffy_rawhide_kid.sql | 2 +- src/actions/system-config.ts | 22 +++++++------------ .../_components/system-settings-form.tsx | 6 ++++- src/app/v1/_lib/proxy/forwarder.ts | 19 ++++++---------- src/lib/api-client/v1/openapi-types.gen.ts | 6 ++--- src/lib/api/v1/schemas/system-config.ts | 6 ++++- src/lib/config/system-settings-cache.ts | 2 +- .../system-settings-discovery.test.ts | 10 +++++++++ 8 files changed, 40 insertions(+), 33 deletions(-) diff --git a/drizzle/0110_daffy_rawhide_kid.sql b/drizzle/0110_daffy_rawhide_kid.sql index f52f66675..ff57e9a44 100644 --- a/drizzle/0110_daffy_rawhide_kid.sql +++ b/drizzle/0110_daffy_rawhide_kid.sql @@ -4,4 +4,4 @@ ALTER TABLE "system_settings" ADD COLUMN "max_discovery_rounds" integer DEFAULT ALTER TABLE "system_settings" ADD COLUMN "discovery_sla_ms" integer DEFAULT 10000 NOT NULL;--> statement-breakpoint ALTER TABLE "system_settings" ADD COLUMN "sticky_sla_ms" integer DEFAULT 20000 NOT NULL;--> statement-breakpoint ALTER TABLE "system_settings" ADD COLUMN "racing_total_timeout_ms" integer DEFAULT 60000 NOT NULL;--> statement-breakpoint -ALTER TABLE "system_settings" ADD COLUMN "sticky_timeout_cooldown_ms" integer DEFAULT 300000 NOT NULL; \ No newline at end of file +ALTER TABLE "system_settings" ADD COLUMN "sticky_timeout_cooldown_ms" integer DEFAULT 300000 NOT NULL; diff --git a/src/actions/system-config.ts b/src/actions/system-config.ts index 2bbf59be8..40e46320b 100644 --- a/src/actions/system-config.ts +++ b/src/actions/system-config.ts @@ -5,6 +5,7 @@ import { locales } from "@/i18n/config"; import { emitActionAudit } from "@/lib/audit/emit"; import { getSession } from "@/lib/auth"; import { invalidateSystemSettingsCache } from "@/lib/config"; +import { DEFAULT_SETTINGS } from "@/lib/config/system-settings-cache"; import { logger } from "@/lib/logger"; import { publishCurrentPublicStatusConfigProjection } from "@/lib/public-status/config-publisher"; import { schedulePublicStatusRebuild } from "@/lib/public-status/rebuild-hints"; @@ -25,12 +26,7 @@ import type { } from "@/types/system-config"; import type { ActionResult } from "./types"; -const DEFAULT_DISCOVERY_WINDOW = { - discoverySlaMs: 10_000, - stickySlaMs: 20_000, - maxDiscoveryRounds: 2, - racingTotalTimeoutMs: 60_000, -} as const; +const DISCOVERY_WINDOW_INVALID = "DISCOVERY_WINDOW_INVALID"; export async function fetchSystemSettings(): Promise> { try { @@ -126,19 +122,16 @@ export async function saveSystemSettings(formData: { const validated = UpdateSystemSettingsSchema.parse(formData); const effectiveDiscoveryWindow = { discoverySlaMs: - validated.discoverySlaMs ?? - before?.discoverySlaMs ?? - DEFAULT_DISCOVERY_WINDOW.discoverySlaMs, - stickySlaMs: - validated.stickySlaMs ?? before?.stickySlaMs ?? DEFAULT_DISCOVERY_WINDOW.stickySlaMs, + validated.discoverySlaMs ?? before?.discoverySlaMs ?? DEFAULT_SETTINGS.discoverySlaMs, + stickySlaMs: validated.stickySlaMs ?? before?.stickySlaMs ?? DEFAULT_SETTINGS.stickySlaMs, maxDiscoveryRounds: validated.maxDiscoveryRounds ?? before?.maxDiscoveryRounds ?? - DEFAULT_DISCOVERY_WINDOW.maxDiscoveryRounds, + DEFAULT_SETTINGS.maxDiscoveryRounds, racingTotalTimeoutMs: validated.racingTotalTimeoutMs ?? before?.racingTotalTimeoutMs ?? - DEFAULT_DISCOVERY_WINDOW.racingTotalTimeoutMs, + DEFAULT_SETTINGS.racingTotalTimeoutMs, }; if ( effectiveDiscoveryWindow.racingTotalTimeoutMs < @@ -147,7 +140,8 @@ export async function saveSystemSettings(formData: { ) { return { ok: false, - error: "竞速总超时必须不小于 Sticky SLA + Discovery 轮数 × Discovery SLA", + error: "Discovery window validation failed.", + errorCode: DISCOVERY_WINDOW_INVALID, }; } const updated = await updateSystemSettings({ diff --git a/src/app/[locale]/settings/config/_components/system-settings-form.tsx b/src/app/[locale]/settings/config/_components/system-settings-form.tsx index ae3307d22..2642cbf8a 100644 --- a/src/app/[locale]/settings/config/_components/system-settings-form.tsx +++ b/src/app/[locale]/settings/config/_components/system-settings-form.tsx @@ -393,7 +393,11 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) }); if (!result.ok) { - toast.error(result.error || t("saveFailed")); + const errorMessage = + result.errorCode === "DISCOVERY_WINDOW_INVALID" + ? t("discoveryWindowInvalid") + : result.error || t("saveFailed"); + toast.error(errorMessage); return; } diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 59de0b133..9965e311a 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -4847,7 +4847,10 @@ export class ProxyForwarder { const concurrency = Math.max(1, Math.floor(settings.discoveryConcurrency ?? 2)); const maxRounds = Math.max(1, Math.floor(settings.maxDiscoveryRounds ?? 2)); const discoverySlaMs = Math.max(1, settings.discoverySlaMs ?? 10_000); - const stickySlaMs = Math.max(discoverySlaMs, settings.stickySlaMs ?? 20_000); + // Respect the configured Sticky budget. The settings validator already + // checks the total pre-winner window; a shorter Sticky SLA is a valid + // deliberate choice and must not be silently expanded at runtime. + const stickySlaMs = Math.max(1, settings.stickySlaMs ?? 20_000); const totalTimeoutMs = Math.max(stickySlaMs, settings.racingTotalTimeoutMs ?? 60_000); const protocol = ProxyForwarder.discoveryProtocol(session); const coordinator = new DiscoveryCoordinator({ concurrency, maxRounds }); @@ -5186,17 +5189,9 @@ export class ProxyForwarder { throw new ProxyError("Invalid upstream discovery response", 502); } if (!validity.ready) continue; - const normalPendingHigher = Array.from(attempts.values()).some( - (other) => - other.pending && - other.kind === "normal" && - (other.provider.priority || 0) < (provider.priority || 0) - ); - // Once a valid prefix is buffered behind a higher-priority normal - // attempt, stop reading. This leaves the reader idle so a later - // fallback promotion can transfer it without racing an in-flight - // read and losing bytes from the buffered response. - if (normalPendingHigher) return; + // Record readiness even when the priority gate holds this attempt. + // The coordinator can then promote the buffered stream if the + // higher-priority attempt fails or the round closes. const action = coordinator.markReady(id); if (action.type === "commit_normal" || action.type === "promote_fallback") await commit(attempt); diff --git a/src/lib/api-client/v1/openapi-types.gen.ts b/src/lib/api-client/v1/openapi-types.gen.ts index ad00a4ecd..4af7d75bd 100644 --- a/src/lib/api-client/v1/openapi-types.gen.ts +++ b/src/lib/api-client/v1/openapi-types.gen.ts @@ -11891,7 +11891,7 @@ export interface operations { discoveryConcurrency: number; /** @description Maximum number of Discovery rounds. */ maxDiscoveryRounds: number; - /** @description 首字节 Discovery SLA in milliseconds. */ + /** @description First-byte Discovery SLA in milliseconds. */ discoverySlaMs: number; /** @description Sticky probe SLA in milliseconds. */ stickySlaMs: number; @@ -12167,7 +12167,7 @@ export interface operations { discoveryConcurrency?: number; /** @description Maximum number of Discovery rounds. */ maxDiscoveryRounds?: number; - /** @description 首字节 Discovery SLA in milliseconds. */ + /** @description First-byte Discovery SLA in milliseconds. */ discoverySlaMs?: number; /** @description Sticky probe SLA in milliseconds. */ stickySlaMs?: number; @@ -12316,7 +12316,7 @@ export interface operations { discoveryConcurrency: number; /** @description Maximum number of Discovery rounds. */ maxDiscoveryRounds: number; - /** @description 首字节 Discovery SLA in milliseconds. */ + /** @description First-byte Discovery SLA in milliseconds. */ discoverySlaMs: number; /** @description Sticky probe SLA in milliseconds. */ stickySlaMs: number; diff --git a/src/lib/api/v1/schemas/system-config.ts b/src/lib/api/v1/schemas/system-config.ts index f27d5993e..cdd5d7a12 100644 --- a/src/lib/api/v1/schemas/system-config.ts +++ b/src/lib/api/v1/schemas/system-config.ts @@ -105,7 +105,11 @@ export const SystemSettingsSchema = z .positive() .describe("Maximum number of normal Discovery attempts in the initial batch."), maxDiscoveryRounds: z.number().int().positive().describe("Maximum number of Discovery rounds."), - discoverySlaMs: z.number().int().positive().describe("首字节 Discovery SLA in milliseconds."), + discoverySlaMs: z + .number() + .int() + .positive() + .describe("First-byte Discovery SLA in milliseconds."), stickySlaMs: z.number().int().positive().describe("Sticky probe SLA in milliseconds."), racingTotalTimeoutMs: z .number() diff --git a/src/lib/config/system-settings-cache.ts b/src/lib/config/system-settings-cache.ts index 0dd733b75..fd3cee750 100644 --- a/src/lib/config/system-settings-cache.ts +++ b/src/lib/config/system-settings-cache.ts @@ -32,7 +32,7 @@ export function getCachedSystemSettingsOnlyCache(): SystemSettings | null { } /** Default settings used when cache fetch fails */ -const DEFAULT_SETTINGS: Pick< +export const DEFAULT_SETTINGS: Pick< SystemSettings, | "enableHttp2" | "enableOpenaiResponsesWebsocket" diff --git a/tests/unit/validation/system-settings-discovery.test.ts b/tests/unit/validation/system-settings-discovery.test.ts index f6b7df030..14841cb4b 100644 --- a/tests/unit/validation/system-settings-discovery.test.ts +++ b/tests/unit/validation/system-settings-discovery.test.ts @@ -26,6 +26,16 @@ describe("UpdateSystemSettingsSchema Discovery settings", () => { ).toThrow("竞速总超时"); }); + it("preserves an intentionally shorter Sticky SLA", () => { + const result = UpdateSystemSettingsSchema.parse({ + discoverySlaMs: 10_000, + stickySlaMs: 5_000, + maxDiscoveryRounds: 2, + racingTotalTimeoutMs: 25_000, + }); + expect(result.stickySlaMs).toBe(5_000); + }); + it("allows partial updates so the server can merge them with stored settings", () => { expect(UpdateSystemSettingsSchema.parse({ discoveryEnabled: true })).toEqual({ discoveryEnabled: true, From 488bc8cd5330c6fbd110d829f661d43141daddb4 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 15:59:22 -0400 Subject: [PATCH 22/89] fix: guard legacy binding fallback races --- src/lib/redis/lua-scripts.ts | 12 ++++ src/lib/redis/session-binding.ts | 66 ++++++++++++++++++++ tests/unit/lib/redis/session-binding.test.ts | 55 ++++++++++++++++ 3 files changed, 133 insertions(+) diff --git a/src/lib/redis/lua-scripts.ts b/src/lib/redis/lua-scripts.ts index 7feb1a763..4bc825b8b 100644 --- a/src/lib/redis/lua-scripts.ts +++ b/src/lib/redis/lua-scripts.ts @@ -4,6 +4,18 @@ * 用于保证 Redis 操作的原子性 */ +/** + * Delete a legacy provider mirror only when it still contains the value that + * the guarded fallback mutation wrote. This is intentionally single-key so it + * remains usable on Redis Cluster when the multi-key binding scripts are not. + */ +export const DELETE_LEGACY_PROVIDER_IF_VALUE = ` +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) +end +return 0 +`; + /** * Atomic concurrency check + session tracking (TC-041 fixed version) * diff --git a/src/lib/redis/session-binding.ts b/src/lib/redis/session-binding.ts index 54a58de5c..eb91306dd 100644 --- a/src/lib/redis/session-binding.ts +++ b/src/lib/redis/session-binding.ts @@ -6,6 +6,7 @@ import { getRedisClient } from "./client"; import { CAS_SESSION_BINDING, CLEAR_SESSION_BINDING, + DELETE_LEGACY_PROVIDER_IF_VALUE, READ_OR_RECONCILE_SESSION_BINDING, TERMINATE_SESSION_BINDING, } from "./lua-scripts"; @@ -834,6 +835,39 @@ function parseLegacyProviderId(raw: string | null): number | null | undefined { return isPositiveInteger(providerId) ? providerId : undefined; } +/** + * A legacy fallback mutation can race a different worker which has already + * recovered versioned binding capability. Re-check the canonical key after a + * legacy write and fail closed if the versioned owner appeared in between. + * Rollback uses a single-key conditional Lua script, so a concurrent versioned + * writer cannot have its newer provider value deleted. If scripts are disabled + * entirely, the mutation still fails closed and leaves the mirror untouched. + */ +async function rejectLegacyMutationAfterCanonicalAppeared( + redis: SessionBindingRedisClient, + keys: SessionBindingKeys, + rollbackProviderValue?: string +): Promise { + if ((await redis.exists(keys.canonical)) === 0) return null; + + if (rollbackProviderValue !== undefined) { + try { + await redis.eval( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + keys.legacyProvider, + rollbackProviderValue + ); + } catch (error) { + logger.warn("Legacy binding rollback could not execute atomically", { + error: error instanceof Error ? error.message : String(error), + }); + } + } + + return conflict("canonical_exists"); +} + export async function mutateLegacySessionBindingSafely( input: LegacySessionBindingMutationInput ): Promise { @@ -899,6 +933,13 @@ export async function mutateLegacySessionBindingSafely( case "refresh": await redis.expire(keys.legacyOwner, ttlSeconds); if (legacyProvider !== null) await redis.expire(keys.legacyProvider, ttlSeconds); + { + const conflictAfterRefresh = await rejectLegacyMutationAfterCanonicalAppeared( + redis, + keys + ); + if (conflictAfterRefresh) return conflictAfterRefresh; + } return { status: "ok", changed: false, providerId: legacyProvider }; case "bind_if_absent": { if (legacyProvider !== null) { @@ -913,6 +954,12 @@ export async function mutateLegacySessionBindingSafely( ); await redis.expire(keys.legacyOwner, ttlSeconds); if (result === "OK") { + const conflictAfterBind = await rejectLegacyMutationAfterCanonicalAppeared( + redis, + keys, + input.mutation.providerId.toString() + ); + if (conflictAfterBind) return conflictAfterBind; return { status: "ok", changed: true, providerId: input.mutation.providerId }; } const concurrentProvider = parseLegacyProviderId(await redis.get(keys.legacyProvider)); @@ -922,6 +969,14 @@ export async function mutateLegacySessionBindingSafely( case "set": await redis.setex(keys.legacyProvider, ttlSeconds, input.mutation.providerId.toString()); await redis.expire(keys.legacyOwner, ttlSeconds); + { + const conflictAfterSet = await rejectLegacyMutationAfterCanonicalAppeared( + redis, + keys, + input.mutation.providerId.toString() + ); + if (conflictAfterSet) return conflictAfterSet; + } return { status: "ok", changed: true, providerId: input.mutation.providerId }; case "clear": if ( @@ -938,6 +993,10 @@ export async function mutateLegacySessionBindingSafely( } await redis.del(keys.legacyProvider); await redis.expire(keys.legacyOwner, ttlSeconds); + { + const conflictAfterClear = await rejectLegacyMutationAfterCanonicalAppeared(redis, keys); + if (conflictAfterClear) return conflictAfterClear; + } return { status: "ok", changed: true, providerId: null }; case "terminate": if ( @@ -948,6 +1007,13 @@ export async function mutateLegacySessionBindingSafely( } if (legacyProvider !== null) await redis.del(keys.legacyProvider); await redis.del(keys.legacyOwner); + { + const conflictAfterTerminate = await rejectLegacyMutationAfterCanonicalAppeared( + redis, + keys + ); + if (conflictAfterTerminate) return conflictAfterTerminate; + } return { status: "ok", changed: legacyProvider !== null, providerId: null }; } } catch (error) { diff --git a/tests/unit/lib/redis/session-binding.test.ts b/tests/unit/lib/redis/session-binding.test.ts index 09ec9072c..b2193e270 100644 --- a/tests/unit/lib/redis/session-binding.test.ts +++ b/tests/unit/lib/redis/session-binding.test.ts @@ -26,6 +26,7 @@ import { import { CAS_SESSION_BINDING, CLEAR_SESSION_BINDING, + DELETE_LEGACY_PROVIDER_IF_VALUE, READ_OR_RECONCILE_SESSION_BINDING, TERMINATE_SESSION_BINDING, } from "@/lib/redis/lua-scripts"; @@ -786,6 +787,60 @@ describe("versioned session binding operations", () => { expect(mock.getMock).not.toHaveBeenCalled(); }); + it("fails closed and rolls back its provider write if canonical state appears mid-mutation", async () => { + let provider: string | null = "8"; + const mock = createMockRedis({ + operationResponses: { + [DELETE_LEGACY_PROVIDER_IF_VALUE]: [ + () => { + provider = null; + return 1; + }, + ], + }, + }); + let existsCalls = 0; + mock.existsMock.mockImplementation(async () => { + existsCalls += 1; + // Initial guard and pre-mutation guard pass; the post-write guard sees + // a versioned worker creating the canonical binding concurrently. + return existsCalls === 3 ? 1 : 0; + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return "4"; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.setexMock.mockImplementation(async (key: string, _ttl: number, value: string) => { + if (key === "session:sid:provider") provider = value; + return "OK"; + }); + mock.delMock.mockImplementation(async (key: string) => { + if (key === "session:sid:provider") provider = null; + return 1; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "set", providerId: 10 }, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "canonical_exists", + legacyFallbackAllowed: false, + }); + expect(provider).toBeNull(); + expect(mock.evalMock).toHaveBeenCalledWith( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + "session:sid:provider", + "10" + ); + }); + it("uses the tenant-authorized termination primitive and leaves a tombstone", async () => { const mock = createMockRedis({ operationResponses: { From e95f0d0555d607248ebc1dce377a99cd657d7e93 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:03:02 -0400 Subject: [PATCH 23/89] test(discovery): cover complete ready stream --- tests/unit/proxy/discovery-validity.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/proxy/discovery-validity.test.ts b/tests/unit/proxy/discovery-validity.test.ts index 83feb4f8e..408fca35a 100644 --- a/tests/unit/proxy/discovery-validity.test.ts +++ b/tests/unit/proxy/discovery-validity.test.ts @@ -74,4 +74,16 @@ describe("discovery validity", () => { error: false, }); }); + + it("keeps ready when content and the terminal marker arrive in one read", () => { + const parser = new DiscoveryValidityParser("openai-chat"); + + expect( + parser.push('data: {"choices":[{"delta":{"content":"done"}}]}\n\ndata: [DONE]\n\n') + ).toEqual({ + ready: true, + terminal: true, + error: false, + }); + }); }); From 7e6dadf9cf85c5fc17a9bab622dbbdf64a167910 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:05:01 -0400 Subject: [PATCH 24/89] fix(proxy): pause held discovery readers --- src/app/v1/_lib/proxy/forwarder.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 7648319c5..97be3effa 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -5161,8 +5161,9 @@ export class ProxyForwarder { // the buffered stream is already sufficient for later promotion. if (validity.terminal) return; // The coordinator owns priority gating. A ready lower-priority - // candidate stays held while a higher tier is still pending. - if (action.type === "none") continue; + // candidate stays held while a higher tier is still pending. Stop + // reading so later chunks are not consumed before promotion. + if (action.type === "none") return; return; } }) From 309d6ca28202f5a2300f525d4b6fadf583094e6e Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:09:33 -0400 Subject: [PATCH 25/89] test(settings): cover discovery window boundary --- .../unit/validation/system-settings-discovery.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/validation/system-settings-discovery.test.ts b/tests/unit/validation/system-settings-discovery.test.ts index 14841cb4b..1950ef225 100644 --- a/tests/unit/validation/system-settings-discovery.test.ts +++ b/tests/unit/validation/system-settings-discovery.test.ts @@ -26,6 +26,17 @@ describe("UpdateSystemSettingsSchema Discovery settings", () => { ).toThrow("竞速总超时"); }); + it("accepts a total deadline exactly equal to the configured discovery window", () => { + expect(() => + UpdateSystemSettingsSchema.parse({ + discoverySlaMs: 10_000, + stickySlaMs: 20_000, + maxDiscoveryRounds: 2, + racingTotalTimeoutMs: 40_000, + }) + ).not.toThrow(); + }); + it("preserves an intentionally shorter Sticky SLA", () => { const result = UpdateSystemSettingsSchema.parse({ discoverySlaMs: 10_000, From 0de55d73442df4db74b47f016106a1aa0b6a1ca1 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:13:04 -0400 Subject: [PATCH 26/89] fix(discovery): recognize Anthropic tool deltas --- src/app/v1/_lib/proxy/discovery-validity.ts | 3 +++ tests/unit/proxy/discovery-validity.test.ts | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/src/app/v1/_lib/proxy/discovery-validity.ts b/src/app/v1/_lib/proxy/discovery-validity.ts index d4ece4eff..7d53493b7 100644 --- a/src/app/v1/_lib/proxy/discovery-validity.ts +++ b/src/app/v1/_lib/proxy/discovery-validity.ts @@ -27,6 +27,9 @@ function hasContent(value: unknown): boolean { "functionCall", "function_call", "arguments", + "partial_json", + "id", + "name", "input", "parts", ].some((key) => hasContent(object[key])); diff --git a/tests/unit/proxy/discovery-validity.test.ts b/tests/unit/proxy/discovery-validity.test.ts index 408fca35a..c39065ebe 100644 --- a/tests/unit/proxy/discovery-validity.test.ts +++ b/tests/unit/proxy/discovery-validity.test.ts @@ -86,4 +86,17 @@ describe("discovery validity", () => { error: false, }); }); + + it("accepts Anthropic tool-use partial JSON as deliverable content", () => { + const parser = new DiscoveryValidityParser("anthropic"); + + expect( + parser.push('data: {"type":"content_block_delta","delta":{"partial_json":"{\\"x\\":1}"}}\n\n') + ).toMatchObject({ ready: true, error: false }); + expect(parser.push('data: {"type":"message_stop"}\n\n')).toMatchObject({ + ready: true, + terminal: true, + error: false, + }); + }); }); From 8e0cedba26e9309b069cc73203e550422a3acf92 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:13:11 -0400 Subject: [PATCH 27/89] fix(proxy): close discovery attempt cleanup gaps --- .../v1/_lib/proxy/discovery-coordinator.ts | 14 +++++++ src/app/v1/_lib/proxy/discovery-validity.ts | 12 +++++- src/app/v1/_lib/proxy/forwarder.ts | 37 ++++++++++++------- .../unit/proxy/discovery-coordinator.test.ts | 15 ++++++++ tests/unit/proxy/discovery-validity.test.ts | 15 ++++++++ ...forwarder-provider-session-release.test.ts | 16 ++++++++ 6 files changed, 94 insertions(+), 15 deletions(-) diff --git a/src/app/v1/_lib/proxy/discovery-coordinator.ts b/src/app/v1/_lib/proxy/discovery-coordinator.ts index a38d0813b..4a94baf70 100644 --- a/src/app/v1/_lib/proxy/discovery-coordinator.ts +++ b/src/app/v1/_lib/proxy/discovery-coordinator.ts @@ -122,6 +122,20 @@ export class DiscoveryCoordinator { return this.chooseReadyNormal(); } + /** Convert a timed-out Sticky attempt into the request's fallback lane. */ + demoteToFallback( + id: string, + requestEpoch = this.requestEpoch, + roundEpoch = this.roundEpoch + ): boolean { + if (!this.acceptsEpoch(requestEpoch, roundEpoch) || this.isTerminal) return false; + const attempt = this.attempts.get(id); + if (!attempt?.pending) return false; + attempt.kind = "fallback"; + this.state = "FALLBACK_READY_HELD"; + return true; + } + markFailed( id: string, requestEpoch = this.requestEpoch, diff --git a/src/app/v1/_lib/proxy/discovery-validity.ts b/src/app/v1/_lib/proxy/discovery-validity.ts index d4ece4eff..198d77699 100644 --- a/src/app/v1/_lib/proxy/discovery-validity.ts +++ b/src/app/v1/_lib/proxy/discovery-validity.ts @@ -28,10 +28,20 @@ function hasContent(value: unknown): boolean { "function_call", "arguments", "input", + "partial_json", "parts", ].some((key) => hasContent(object[key])); } +function hasAnthropicContentBlock(value: unknown): boolean { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const block = value as Record; + if (typeof block.type !== "string" || block.type.length === 0) return false; + // Text blocks need non-empty text; tool_use/thinking/image blocks are + // deliverable as soon as their typed block starts, even with empty input. + return block.type === "text" ? hasContent(block.text) : true; +} + function classifyJson(value: unknown, protocol: DiscoveryProtocol): DiscoveryValidity { if (!value || typeof value !== "object") return { ready: false, terminal: false, error: true }; const object = value as Record; @@ -89,7 +99,7 @@ function classifyJson(value: unknown, protocol: DiscoveryProtocol): DiscoveryVal return { ready: (object.type === "content_block_delta" && hasContent(object.delta)) || - (object.type === "content_block_start" && hasContent(object.content_block)) || + (object.type === "content_block_start" && hasAnthropicContentBlock(object.content_block)) || hasContent(object.content), terminal: false, error: false, diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 97be3effa..fdf0de21c 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -4897,12 +4897,7 @@ export class ProxyForwarder { }); const releaseProviderRef = (providerId: number) => { - if (!session.sessionId) return; - const consumer = (session as { consumeProviderSessionRef?: (id: number) => boolean }) - .consumeProviderSessionRef; - if (consumer?.call(session, providerId)) { - void RateLimitService.releaseProviderSession(providerId, session.sessionId); - } + ProxyForwarder.releaseProviderSessionRef(session, providerId); }; const cancelAttempt = (attempt: (typeof winner & { id: string }) | null, reason: string) => { @@ -5035,6 +5030,7 @@ export class ProxyForwarder { const launch = async (provider: Provider, kind: "normal" | "fallback"): Promise => { if (settled || committed || launched.has(provider.id)) return; launched.add(provider.id); + let providerSessionRefRecorded = false; if (provider.id !== initialProvider.id && session.sessionId) { const limit = provider.limitConcurrentSessions || 0; const check = await RateLimitService.checkAndTrackProviderSession( @@ -5046,9 +5042,19 @@ export class ProxyForwarder { launched.delete(provider.id); throw new ProxyError(check.reason || "Provider concurrent limit reached", 503); } - if (check.referenced) session.recordProviderSessionRef(provider.id); + if (check.referenced) { + session.recordProviderSessionRef(provider.id); + providerSessionRefRecorded = true; + } + } + let endpoint: Awaited>; + try { + endpoint = await ProxyForwarder.resolveStreamingHedgeEndpoint(session, provider); + } catch (error) { + launched.delete(provider.id); + if (providerSessionRefRecorded) releaseProviderRef(provider.id); + throw error; } - const endpoint = await ProxyForwarder.resolveStreamingHedgeEndpoint(session, provider); const attemptSession = provider.id === initialProvider.id ? session @@ -5330,6 +5336,7 @@ export class ProxyForwarder { (attempt) => attempt.pending && attempt.provider.id === initialProvider.id ); if (sticky) { + if (!coordinator.demoteToFallback(sticky.id)) return; sticky.kind = "fallback"; if (bindingSnapshot && bindingSnapshot.providerId === initialProvider.id) { void SessionManager.clearVersionedSessionProvider( @@ -5641,15 +5648,17 @@ export class ProxyForwarder { return; } + ProxyForwarder.releaseProviderSessionRef(session, providerId); + } + + private static releaseProviderSessionRef(session: ProxySession, providerId: number): boolean { + if (!session.sessionId) return false; const providerSessionRefConsumer = ( - session as { consumeProviderSessionRef?: (providerId: number) => boolean } + session as { consumeProviderSessionRef?: (id: number) => boolean } ).consumeProviderSessionRef; - - if (!providerSessionRefConsumer?.call(session, providerId)) { - return; - } - + if (!providerSessionRefConsumer?.call(session, providerId)) return false; void RateLimitService.releaseProviderSession(providerId, session.sessionId); + return true; } private static buildAllProvidersUnavailableError(finalError?: Error | null): ProxyError { diff --git a/tests/unit/proxy/discovery-coordinator.test.ts b/tests/unit/proxy/discovery-coordinator.test.ts index 328a02e97..3a41a8d5e 100644 --- a/tests/unit/proxy/discovery-coordinator.test.ts +++ b/tests/unit/proxy/discovery-coordinator.test.ts @@ -53,6 +53,21 @@ describe("DiscoveryCoordinator", () => { expect(coordinator.snapshot.find((item) => item.id === "a")?.kind).toBe("fallback"); }); + it("keeps Sticky demotion synchronized with the coordinator", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("sticky", 1)); + + expect(coordinator.demoteToFallback("sticky")).toBe(true); + expect(coordinator.snapshot.find((item) => item.id === "sticky")).toMatchObject({ + kind: "fallback", + pending: true, + }); + expect(coordinator.markReady("sticky")).toEqual({ + type: "promote_fallback", + attemptId: "sticky", + }); + }); + it("chooses the best ready normal at a boundary", () => { const coordinator = new DiscoveryCoordinator({ concurrency: 3, maxRounds: 1 }); coordinator.addAttempt(attempt("a", 10)); diff --git a/tests/unit/proxy/discovery-validity.test.ts b/tests/unit/proxy/discovery-validity.test.ts index 408fca35a..7a6f95fc4 100644 --- a/tests/unit/proxy/discovery-validity.test.ts +++ b/tests/unit/proxy/discovery-validity.test.ts @@ -86,4 +86,19 @@ describe("discovery validity", () => { error: false, }); }); + + it("accepts Anthropic tool-use starts and partial JSON deltas", () => { + expect( + classifyDiscoveryChunk( + 'data: {"type":"content_block_start","content_block":{"type":"tool_use","id":"tu_1","name":"search","input":{}}}\n', + "anthropic" + ).ready + ).toBe(true); + expect( + classifyDiscoveryChunk( + 'data: {"type":"content_block_delta","delta":{"type":"input_json_delta","partial_json":"{\\"q\\":1}"}}\n', + "anthropic" + ).ready + ).toBe(true); + }); }); diff --git a/tests/unit/proxy/proxy-forwarder-provider-session-release.test.ts b/tests/unit/proxy/proxy-forwarder-provider-session-release.test.ts index efc8a9028..0fa7ebafa 100644 --- a/tests/unit/proxy/proxy-forwarder-provider-session-release.test.ts +++ b/tests/unit/proxy/proxy-forwarder-provider-session-release.test.ts @@ -68,6 +68,22 @@ describe("ProxyForwarder provider failure session release", () => { expect(mocks.releaseProviderSession).not.toHaveBeenCalled(); }); + it("endpoint resolution rollback releases only a recorded provider ref", async () => { + const { ProxyForwarder } = await import("@/app/v1/_lib/proxy/forwarder"); + const forwarderInternals = ProxyForwarder as unknown as { + releaseProviderSessionRef: (session: ProxySession, providerId: number) => boolean; + }; + const consumeProviderSessionRef = vi.fn(() => true); + const session = { + sessionId: "sess_endpoint_failure", + consumeProviderSessionRef, + } as unknown as ProxySession; + + expect(forwarderInternals.releaseProviderSessionRef(session, 42)).toBe(true); + expect(consumeProviderSessionRef).toHaveBeenCalledWith(42); + expect(mocks.releaseProviderSession).toHaveBeenCalledWith(42, "sess_endpoint_failure"); + }); + it("重复标记同一供应商时只释放一次，避免 hedge 路径重复 ZREM", async () => { const { ProxyForwarder } = await import("@/app/v1/_lib/proxy/forwarder"); const forwarderInternals = ProxyForwarder as unknown as { From 09fbb57371359a875d0609229f9222fce5ab6f68 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:18:48 -0400 Subject: [PATCH 28/89] fix: preserve legacy mirror during binding races --- src/lib/redis/lua-scripts.ts | 13 +++++ src/lib/redis/session-binding.ts | 27 +++++++++- src/lib/session-manager.ts | 7 --- tests/unit/lib/redis/session-binding.test.ts | 52 ++++++++++++++++++++ 4 files changed, 90 insertions(+), 9 deletions(-) diff --git a/src/lib/redis/lua-scripts.ts b/src/lib/redis/lua-scripts.ts index 4bc825b8b..342cf35cb 100644 --- a/src/lib/redis/lua-scripts.ts +++ b/src/lib/redis/lua-scripts.ts @@ -16,6 +16,19 @@ end return 0 `; +/** + * Restore a legacy provider mirror only when the guarded fallback clear left + * it absent. The conditional write keeps a concurrent versioned writer's + * newer mirror value intact. + */ +export const RESTORE_LEGACY_PROVIDER_IF_ABSENT = ` +if redis.call('EXISTS', KEYS[1]) == 0 then + redis.call('SETEX', KEYS[1], ARGV[2], ARGV[1]) + return 1 +end +return 0 +`; + /** * Atomic concurrency check + session tracking (TC-041 fixed version) * diff --git a/src/lib/redis/session-binding.ts b/src/lib/redis/session-binding.ts index eb91306dd..1fd7ae97b 100644 --- a/src/lib/redis/session-binding.ts +++ b/src/lib/redis/session-binding.ts @@ -8,6 +8,7 @@ import { CLEAR_SESSION_BINDING, DELETE_LEGACY_PROVIDER_IF_VALUE, READ_OR_RECONCILE_SESSION_BINDING, + RESTORE_LEGACY_PROVIDER_IF_ABSENT, TERMINATE_SESSION_BINDING, } from "./lua-scripts"; @@ -846,7 +847,8 @@ function parseLegacyProviderId(raw: string | null): number | null | undefined { async function rejectLegacyMutationAfterCanonicalAppeared( redis: SessionBindingRedisClient, keys: SessionBindingKeys, - rollbackProviderValue?: string + rollbackProviderValue?: string, + restoreProviderValue?: { value: string; ttlSeconds: number } ): Promise { if ((await redis.exists(keys.canonical)) === 0) return null; @@ -865,6 +867,22 @@ async function rejectLegacyMutationAfterCanonicalAppeared( } } + if (restoreProviderValue !== undefined) { + try { + await redis.eval( + RESTORE_LEGACY_PROVIDER_IF_ABSENT, + 1, + keys.legacyProvider, + restoreProviderValue.value, + restoreProviderValue.ttlSeconds.toString() + ); + } catch (error) { + logger.warn("Legacy binding mirror restoration could not execute atomically", { + error: error instanceof Error ? error.message : String(error), + }); + } + } + return conflict("canonical_exists"); } @@ -994,7 +1012,12 @@ export async function mutateLegacySessionBindingSafely( await redis.del(keys.legacyProvider); await redis.expire(keys.legacyOwner, ttlSeconds); { - const conflictAfterClear = await rejectLegacyMutationAfterCanonicalAppeared(redis, keys); + const conflictAfterClear = await rejectLegacyMutationAfterCanonicalAppeared( + redis, + keys, + undefined, + { value: legacyProvider.toString(), ttlSeconds } + ); if (conflictAfterClear) return conflictAfterClear; } return { status: "ok", changed: true, providerId: null }; diff --git a/src/lib/session-manager.ts b/src/lib/session-manager.ts index 3cadabd6e..02291f1c6 100644 --- a/src/lib/session-manager.ts +++ b/src/lib/session-manager.ts @@ -2795,17 +2795,10 @@ export class SessionManager { redis, }); if (terminated.status !== "ok") { - const latest = await readOrReconcileSessionBinding({ - sessionId, - keyId, - ttlSeconds: SessionManager.SESSION_TTL, - redis, - }); logger.warn("SessionManager: Versioned session termination blocked", { sessionId, keyId, reason: terminated.reason, - latestStatus: latest.status, }); return false; } diff --git a/tests/unit/lib/redis/session-binding.test.ts b/tests/unit/lib/redis/session-binding.test.ts index b2193e270..11deb4c84 100644 --- a/tests/unit/lib/redis/session-binding.test.ts +++ b/tests/unit/lib/redis/session-binding.test.ts @@ -28,6 +28,7 @@ import { CLEAR_SESSION_BINDING, DELETE_LEGACY_PROVIDER_IF_VALUE, READ_OR_RECONCILE_SESSION_BINDING, + RESTORE_LEGACY_PROVIDER_IF_ABSENT, TERMINATE_SESSION_BINDING, } from "@/lib/redis/lua-scripts"; @@ -841,6 +842,57 @@ describe("versioned session binding operations", () => { ); }); + it("restores a cleared provider mirror if canonical state appears before the post-check", async () => { + let provider: string | null = "8"; + const mock = createMockRedis({ + operationResponses: { + [RESTORE_LEGACY_PROVIDER_IF_ABSENT]: [ + () => { + provider = "8"; + return 1; + }, + ], + }, + }); + let existsCalls = 0; + mock.existsMock.mockImplementation(async () => { + existsCalls += 1; + // Initial guard and pre-mutation guard pass; the post-clear guard sees + // a versioned worker creating canonical state concurrently. + return existsCalls === 3 ? 1 : 0; + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return "4"; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.delMock.mockImplementation(async (key: string) => { + if (key === "session:sid:provider") provider = null; + return 1; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "clear", expectedProviderId: 8 }, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "canonical_exists", + legacyFallbackAllowed: false, + }); + expect(provider).toBe("8"); + expect(mock.evalMock).toHaveBeenCalledWith( + RESTORE_LEGACY_PROVIDER_IF_ABSENT, + 1, + "session:sid:provider", + "8", + "300" + ); + }); + it("uses the tenant-authorized termination primitive and leaves a tombstone", async () => { const mock = createMockRedis({ operationResponses: { From 45dac9476dddab3f6dc1fdbc9ecd8b8fb36e6fb3 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:22:06 -0400 Subject: [PATCH 29/89] fix: validate canonical provider before mirror restore --- src/lib/redis/session-binding.ts | 22 ++++++---- tests/unit/lib/redis/session-binding.test.ts | 42 ++++++++++++++++++++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/src/lib/redis/session-binding.ts b/src/lib/redis/session-binding.ts index 1fd7ae97b..d053f63e4 100644 --- a/src/lib/redis/session-binding.ts +++ b/src/lib/redis/session-binding.ts @@ -57,6 +57,7 @@ export interface SessionBindingRedisClient { eval(script: string, numberOfKeys: number, ...args: Array): Promise; evalsha?(sha1: string, numberOfKeys: number, ...args: Array): Promise; get(key: string): Promise; + hget?(key: string, field: string): Promise; del(...keys: string[]): Promise; exists(key: string): Promise; expire(key: string, ttlSeconds: number): Promise; @@ -869,13 +870,20 @@ async function rejectLegacyMutationAfterCanonicalAppeared( if (restoreProviderValue !== undefined) { try { - await redis.eval( - RESTORE_LEGACY_PROVIDER_IF_ABSENT, - 1, - keys.legacyProvider, - restoreProviderValue.value, - restoreProviderValue.ttlSeconds.toString() - ); + // The canonical hash and legacy mirror intentionally use different + // cluster slots. Read the canonical provider first, then use a + // single-key conditional write so we only restore a mirror that still + // belongs to the canonical binding observed by this fallback path. + const canonicalProvider = await redis.hget?.(keys.canonical, "provider_id"); + if (canonicalProvider === restoreProviderValue.value) { + await redis.eval( + RESTORE_LEGACY_PROVIDER_IF_ABSENT, + 1, + keys.legacyProvider, + restoreProviderValue.value, + restoreProviderValue.ttlSeconds.toString() + ); + } } catch (error) { logger.warn("Legacy binding mirror restoration could not execute atomically", { error: error instanceof Error ? error.message : String(error), diff --git a/tests/unit/lib/redis/session-binding.test.ts b/tests/unit/lib/redis/session-binding.test.ts index 11deb4c84..58fc28c54 100644 --- a/tests/unit/lib/redis/session-binding.test.ts +++ b/tests/unit/lib/redis/session-binding.test.ts @@ -85,6 +85,7 @@ function createMockRedis(options: MockRedisOptions = {}) { if (probeCooldowns.has(key)) return probeCooldowns.get(key) ?? null; return options.cooldownValue ?? null; }); + const hgetMock = vi.fn(async (_key: string, _field: string) => null as string | null); const delMock = vi.fn(async (..._keys: string[]) => { if (cleanupFails) throw new Error("cleanup failed"); return 4; @@ -118,6 +119,7 @@ function createMockRedis(options: MockRedisOptions = {}) { eval: evalMock, ...(options.evalSha ? { evalsha: evalShaMock } : {}), get: getMock, + hget: hgetMock, del: delMock, exists: existsMock, expire: expireMock, @@ -132,6 +134,7 @@ function createMockRedis(options: MockRedisOptions = {}) { evalMock, evalShaMock, getMock, + hgetMock, delMock, existsMock, expireMock, @@ -866,6 +869,7 @@ describe("versioned session binding operations", () => { if (key === "session:sid:provider") return provider; return null; }); + mock.hgetMock.mockResolvedValue("8"); mock.delMock.mockImplementation(async (key: string) => { if (key === "session:sid:provider") provider = null; return 1; @@ -893,6 +897,44 @@ describe("versioned session binding operations", () => { ); }); + it("does not restore a mirror when the concurrent canonical binding is a null tombstone", async () => { + let provider: string | null = "8"; + const mock = createMockRedis(); + let existsCalls = 0; + mock.existsMock.mockImplementation(async () => { + existsCalls += 1; + return existsCalls === 3 ? 1 : 0; + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return "4"; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.hgetMock.mockResolvedValue(null); + mock.delMock.mockImplementation(async (key: string) => { + if (key === "session:sid:provider") provider = null; + return 1; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "clear", expectedProviderId: 8 }, + }); + + expect(result).toMatchObject({ status: "conflict", reason: "canonical_exists" }); + expect(provider).toBeNull(); + expect(mock.evalMock).not.toHaveBeenCalledWith( + RESTORE_LEGACY_PROVIDER_IF_ABSENT, + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything() + ); + }); + it("uses the tenant-authorized termination primitive and leaves a tombstone", async () => { const mock = createMockRedis({ operationResponses: { From 0f7e1701f6882b3d94cb17a89cd7f6d8ff1a7d27 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:27:18 -0400 Subject: [PATCH 30/89] fix(proxy): execute discovery normal winner actions --- src/app/v1/_lib/proxy/forwarder.ts | 1 + .../proxy-forwarder-hedge-first-byte.test.ts | 91 +++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index fdf0de21c..df00d6009 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -5195,6 +5195,7 @@ export class ProxyForwarder { attempt.releaseAgent?.(); releaseProviderRef(provider.id); const actionOwnsNextStep = + failureAction.type === "commit_normal" || failureAction.type === "promote_fallback" || failureAction.type === "launch" || failureAction.type === "terminal_failure"; diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index fe5d7f4a8..1458d825d 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -3,6 +3,7 @@ import { resolveEndpointPolicy } from "@/app/v1/_lib/proxy/endpoint-policy"; const mocks = vi.hoisted(() => ({ pickRandomProviderWithExclusion: vi.fn(), + pickDiscoveryProviders: vi.fn(), recordSuccess: vi.fn(), recordFailure: vi.fn(async () => {}), getCircuitState: vi.fn(() => "closed"), @@ -37,6 +38,7 @@ const mocks = vi.hoisted(() => ({ storeSessionSpecialSettings: vi.fn(async () => {}), storeSessionRequestPhaseSnapshot: vi.fn(async () => {}), storeSessionResponsePhaseSnapshot: vi.fn(async () => {}), + getVersionedBindingCapabilityState: vi.fn(() => "available"), })); vi.mock("@/lib/logger", () => ({ @@ -90,6 +92,7 @@ vi.mock("@/lib/rate-limit/service", () => ({ vi.mock("@/lib/session-manager", () => ({ SessionManager: { + getVersionedBindingCapabilityState: mocks.getVersionedBindingCapabilityState, updateSessionBindingSmart: mocks.updateSessionBindingSmart, updateSessionProvider: mocks.updateSessionProvider, clearSessionProvider: mocks.clearSessionProvider, @@ -103,6 +106,7 @@ vi.mock("@/lib/session-manager", () => ({ vi.mock("@/app/v1/_lib/proxy/provider-selector", () => ({ ProxyProviderResolver: { pickRandomProviderWithExclusion: mocks.pickRandomProviderWithExclusion, + pickDiscoveryProviders: mocks.pickDiscoveryProviders, }, })); @@ -333,6 +337,10 @@ function withThinkingBlocks(session: ProxySession): void { describe("ProxyForwarder - first-byte hedge scheduling", () => { beforeEach(() => { vi.clearAllMocks(); + mocks.getCachedSystemSettings.mockResolvedValue({ + enableThinkingSignatureRectifier: true, + enableThinkingBudgetRectifier: true, + }); mocks.checkAndTrackProviderSession.mockResolvedValue({ allowed: true, count: 1, @@ -2061,6 +2069,89 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { } }); + test("Discovery commits a lower-priority ready stream when the higher tier fails", async () => { + vi.useFakeTimers(); + + try { + const high = createProvider({ id: 1, name: "high", priority: 1 }); + const low = createProvider({ id: 2, name: "low", priority: 10 }); + const session = createSession(); + session.setProvider(high); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 100, + stickySlaMs: 100, + racingTotalTimeoutMs: 500, + stickyTimeoutCooldownMs: 300_000, + }); + mocks.pickDiscoveryProviders.mockResolvedValueOnce([low]); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + + doForward.mockImplementationOnce( + async (_attemptSession, _provider, _baseUrl, _audit, _count, _stream, signal) => { + await new Promise((_resolve, reject) => { + const timer = setTimeout(() => reject(new Error("high tier failed")), 30); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + reject(new Error("high tier aborted")); + }, + { once: true } + ); + }); + } + ); + doForward.mockImplementationOnce( + async (_attemptSession, _provider, _baseUrl, _audit, _count, _stream, signal) => { + const stream = new ReadableStream({ + start(controller) { + const timer = setTimeout(() => { + controller.enqueue( + new TextEncoder().encode( + 'data: {"type":"content_block_delta","delta":{"text":"low"}}\n\n' + ) + ); + controller.close(); + }, 5); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + controller.close(); + }, + { once: true } + ); + }, + }); + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } + ); + + const responsePromise = ProxyForwarder.send(session); + await vi.advanceTimersByTimeAsync(10); + await vi.advanceTimersByTimeAsync(30); + const response = await responsePromise; + + expect(await response.text()).toContain('"low"'); + expect(session.provider?.id).toBe(low.id); + expect(doForward).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + test("removes streaming hedge client abort listener after winner response is returned", async () => { const clientAbortController = new AbortController(); const addSpy = vi.spyOn(clientAbortController.signal, "addEventListener"); From 059de384037785af71628da8d71865bf6ff82263 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:30:18 -0400 Subject: [PATCH 31/89] fix(discovery): preserve binding safety and tool prefixes --- src/app/v1/_lib/proxy/discovery-validity.ts | 1 + src/app/v1/_lib/proxy/forwarder.ts | 9 ++ src/app/v1/_lib/proxy/response-handler.ts | 53 +++++++-- tests/unit/proxy/discovery-validity.test.ts | 12 ++ ...handler-endpoint-circuit-isolation.test.ts | 109 ++++++++++++++++++ 5 files changed, 174 insertions(+), 10 deletions(-) diff --git a/src/app/v1/_lib/proxy/discovery-validity.ts b/src/app/v1/_lib/proxy/discovery-validity.ts index 7d53493b7..21f91c005 100644 --- a/src/app/v1/_lib/proxy/discovery-validity.ts +++ b/src/app/v1/_lib/proxy/discovery-validity.ts @@ -26,6 +26,7 @@ function hasContent(value: unknown): boolean { "tool_calls", "functionCall", "function_call", + "function", "arguments", "partial_json", "id", diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 9965e311a..864d43313 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -5225,6 +5225,15 @@ export class ProxyForwarder { } attempt.releaseAgent?.(); releaseProviderRef(provider.id); + if (lastErrorCategory === ErrorCategory.NON_RETRYABLE_CLIENT_ERROR) { + // Client/input errors are independent of the selected provider. + // Stop Discovery immediately so the same invalid request is not + // fanned out or masked by a later generic fallback error. + await settleFailure( + ProxyForwarder.resolveHedgeTerminalError(lastError, lastErrorCategory) + ); + return; + } const actionOwnsNextStep = failureAction.type === "commit_normal" || failureAction.type === "promote_fallback" || diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index bc36a1a39..3b7ee3d7f 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -15,6 +15,7 @@ import { RateLimitService } from "@/lib/rate-limit"; import { deleteLiveChain } from "@/lib/redis/live-chain-store"; import { SessionManager } from "@/lib/session-manager"; import { SessionTracker } from "@/lib/session-tracker"; +import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import { CODEX_1M_CONTEXT_TOKEN_THRESHOLD } from "@/lib/special-attributes"; import type { CostBreakdown, @@ -1152,16 +1153,55 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; if (meta?.bindingIntent === "none") return; if (meta?.bindingSnapshot && keyId != null) { - await SessionManager.clearVersionedSessionProvider( + const cleared = await SessionManager.clearVersionedSessionProvider( meta.bindingSnapshot, - providerIdForPersistence, + meta.bindingSnapshot.providerId, 0 ); + if (cleared.status !== "ok") { + logger.debug("[ResponseHandler] Discovery binding clear skipped", { + sessionId: meta.bindingSnapshot.sessionId, + keyId: meta.bindingSnapshot.keyId, + expectedProviderId: meta.bindingSnapshot.providerId, + reason: cleared.reason, + }); + } return; } await SessionManager.clearSessionProvider(session.sessionId, providerIdForPersistence, keyId); }; + const compareAndSetDiscoveryBinding = async ( + snapshot: SessionBindingSnapshot, + providerId: number, + keyId: number + ) => { + if (!snapshot) + return { updated: false, reason: "missing_snapshot", details: "missing_snapshot" }; + + let cas = await SessionManager.compareAndSetSessionProvider(snapshot, providerId); + if (cas.status === "conflict" && cas.reason === "canonical_missing") { + // A long stream can outlive SESSION_TTL. Reconcile once before giving up, + // but only retry when the binding still represents the same owner/provider + // observed before the stream. This preserves CAS protection against a + // concurrent request that established a different Sticky in the meantime. + const refreshed = await SessionManager.getSessionBindingSnapshot( + session.sessionId ?? snapshot.sessionId, + keyId + ); + if (refreshed.status === "ok" && refreshed.snapshot.providerId === snapshot.providerId) { + session.setSessionBindingSnapshot(refreshed.snapshot); + cas = await SessionManager.compareAndSetSessionProvider(refreshed.snapshot, providerId); + } + } + + return { + updated: cas.status === "ok", + reason: cas.status === "ok" ? "discovery_generation_cas" : cas.reason, + details: cas.status, + }; + }; + const isHedgeWinner = meta?.isHedgeWinner === true; const billHedgeLosers = meta?.billHedgeLosers === true; @@ -1545,14 +1585,7 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; const result = meta.bindingSnapshot && keyId != null - ? await SessionManager.compareAndSetSessionProvider( - meta.bindingSnapshot, - meta.providerId - ).then((cas) => ({ - updated: cas.status === "ok", - reason: cas.status === "ok" ? "discovery_generation_cas" : cas.reason, - details: cas.status, - })) + ? await compareAndSetDiscoveryBinding(meta.bindingSnapshot, meta.providerId, keyId) : await SessionManager.updateSessionBindingSmart( session.sessionId, meta.providerId, diff --git a/tests/unit/proxy/discovery-validity.test.ts b/tests/unit/proxy/discovery-validity.test.ts index c39065ebe..392a2a75a 100644 --- a/tests/unit/proxy/discovery-validity.test.ts +++ b/tests/unit/proxy/discovery-validity.test.ts @@ -99,4 +99,16 @@ describe("discovery validity", () => { error: false, }); }); + + it("accepts nested OpenAI Chat tool-call arguments", () => { + expect( + parserForOpenAIChatToolCall().push( + 'data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"{\\"x\\":1}"}}]}}]}\n\n' + ) + ).toMatchObject({ ready: true, error: false }); + }); }); + +function parserForOpenAIChatToolCall(): DiscoveryValidityParser { + return new DiscoveryValidityParser("openai-chat"); +} diff --git a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts index af432f126..e6c2ec026 100644 --- a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts +++ b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts @@ -77,6 +77,9 @@ vi.mock("@/lib/session-manager", () => ({ updateSessionUsage: vi.fn(), storeSessionResponse: vi.fn(), clearSessionProvider: vi.fn(), + clearVersionedSessionProvider: vi.fn(), + compareAndSetSessionProvider: vi.fn(), + getSessionBindingSnapshot: vi.fn(), extractCodexPromptCacheKey: vi.fn(), updateSessionBindingSmart: vi.fn(), updateSessionProvider: vi.fn(), @@ -335,6 +338,23 @@ function createSuccessStreamResponse(): Response { }); } +function createSuccessStreamResponseWithCompletion(): Response { + const sseText = + `data: ${JSON.stringify({ type: "content_block_delta", delta: { text: "ok" } })}\n\n` + + `data: ${JSON.stringify({ type: "message_stop" })}\n\n`; + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sseText)); + controller.close(); + }, + }); + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} + async function drainAsyncTasks(): Promise { while (asyncTasks.length > 0) { const tasks = asyncTasks.splice(0, asyncTasks.length); @@ -366,6 +386,39 @@ function setupCommonMocks() { vi.mocked(updateMessageRequestDuration).mockResolvedValue(undefined); vi.mocked(SessionManager.storeSessionResponse).mockResolvedValue(undefined); vi.mocked(SessionManager.clearSessionProvider).mockResolvedValue(undefined); + vi.mocked(SessionManager.clearVersionedSessionProvider).mockResolvedValue({ + status: "ok", + source: "cleared", + snapshot: { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "cleared", + }, + legacyFallbackAllowed: false, + }); + vi.mocked(SessionManager.compareAndSetSessionProvider).mockResolvedValue({ + status: "ok", + source: "updated", + snapshot: { + sessionId: "fake-session", + keyId: 456, + providerId: 1, + generation: "updated", + }, + legacyFallbackAllowed: false, + }); + vi.mocked(SessionManager.getSessionBindingSnapshot).mockResolvedValue({ + status: "ok", + source: "existing", + snapshot: { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "fresh", + }, + legacyFallbackAllowed: false, + }); vi.mocked(SessionManager.updateSessionUsage).mockResolvedValue(undefined); vi.mocked(SessionManager.updateSessionBindingSmart).mockResolvedValue({ updated: true, @@ -537,4 +590,60 @@ describe("Endpoint circuit breaker isolation", () => { expect(mockRecordEndpointSuccess).not.toHaveBeenCalled(); expect(mockRecordEndpointFailure).not.toHaveBeenCalled(); }); + + it("reconciles an expired Discovery binding snapshot before retrying CAS", async () => { + const session = createSession(); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "expired-generation", + } as const; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: snapshot, + requiresCompletionMarker: true, + }); + vi.mocked(SessionManager.compareAndSetSessionProvider) + .mockResolvedValueOnce({ + status: "conflict", + reason: "canonical_missing", + legacyFallbackAllowed: false, + }) + .mockResolvedValueOnce({ + status: "ok", + source: "updated", + snapshot: { + ...snapshot, + providerId: 1, + generation: "renewed-generation", + }, + legacyFallbackAllowed: false, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createSuccessStreamResponseWithCompletion() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.getSessionBindingSnapshot).toHaveBeenCalledWith("fake-session", 456); + expect(SessionManager.compareAndSetSessionProvider).toHaveBeenNthCalledWith(1, snapshot, 1); + expect(SessionManager.compareAndSetSessionProvider).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ generation: "fresh", providerId: null }), + 1 + ); + }); }); From 0690278f9e17539b6a611359c924fecff6e60526 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:32:17 -0400 Subject: [PATCH 32/89] fix(proxy): keep discovery timers and candidates consistent --- src/app/v1/_lib/proxy/forwarder.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index df00d6009..91d78cdaf 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -3845,7 +3845,6 @@ export class ProxyForwarder { disableStreamingHedge?: boolean; }; return ( - settings.discoveryEnabled === true && endpointPolicy.allowRetry && endpointPolicy.allowProviderSwitch && message?.stream === true && @@ -4890,6 +4889,12 @@ export class ProxyForwarder { let totalTimer: NodeJS.Timeout | null = null; let roundTimer: NodeJS.Timeout | null = null; let stickyTimer: NodeJS.Timeout | null = null; + const clearRoundTimer = () => { + if (roundTimer) { + clearTimeout(roundTimer); + roundTimer = null; + } + }; let executeCoordinatorAction: (action: DiscoveryAction) => Promise = async () => {}; let resolveResult: ((result: { response?: Response; error?: Error }) => void) | null = null; const resultPromise = new Promise<{ response?: Response; error?: Error }>((resolve) => { @@ -5225,6 +5230,7 @@ export class ProxyForwarder { const launchNextRound = async () => { if (settled || committed) return; + clearRoundTimer(); currentRound += 1; if (currentRound > maxRounds) return; coordinator.beginRound(); @@ -5234,10 +5240,10 @@ export class ProxyForwarder { await launch(candidate, "normal"); } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); - noMoreCandidates = true; } } if (!committed && !settled) { + clearRoundTimer(); roundTimer = setTimeout(() => { void onBoundary().catch((error) => logger.warn("[Discovery] Round boundary failed", { error }) @@ -5365,14 +5371,15 @@ export class ProxyForwarder { initial, Array.from(launched) ); + if (candidates.length === 0) noMoreCandidates = true; for (const provider of candidates) { try { await launch(provider, "normal"); } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); - noMoreCandidates = true; } } + clearRoundTimer(); roundTimer = setTimeout(() => { void onBoundary().catch((error) => logger.warn("[Discovery] Round boundary failed", { error }) @@ -5385,7 +5392,7 @@ export class ProxyForwarder { } finally { cleanupAbort(); if (totalTimer) clearTimeout(totalTimer); - if (roundTimer) clearTimeout(roundTimer); + clearRoundTimer(); if (stickyTimer) clearTimeout(stickyTimer); } } From 7d49cde3394dc5bd782fc2c0731916f3ed18e7ff Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:36:29 -0400 Subject: [PATCH 33/89] fix(discovery): clear binding against snapshot provider --- src/app/v1/_lib/proxy/response-handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index bc36a1a39..70d0909b5 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -1154,7 +1154,7 @@ function finalizeDeferredStreamingFinalizationIfNeeded( if (meta?.bindingSnapshot && keyId != null) { await SessionManager.clearVersionedSessionProvider( meta.bindingSnapshot, - providerIdForPersistence, + meta.bindingSnapshot.providerId, 0 ); return; From 9ac3d996ad81d9373e62c62684d1074292cc21ef Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:40:56 -0400 Subject: [PATCH 34/89] fix(binding): preserve imported legacy mirrors --- src/lib/redis/session-binding.ts | 19 +++++--- tests/unit/lib/redis/session-binding.test.ts | 49 ++++++++++++++++++++ 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/lib/redis/session-binding.ts b/src/lib/redis/session-binding.ts index d053f63e4..1076b4704 100644 --- a/src/lib/redis/session-binding.ts +++ b/src/lib/redis/session-binding.ts @@ -855,12 +855,19 @@ async function rejectLegacyMutationAfterCanonicalAppeared( if (rollbackProviderValue !== undefined) { try { - await redis.eval( - DELETE_LEGACY_PROVIDER_IF_VALUE, - 1, - keys.legacyProvider, - rollbackProviderValue - ); + // A recovered versioned worker may have imported this exact provider + // into canonical before the post-write check. In that case the legacy + // mirror is already part of valid dual-write state and must survive the + // rollback; deleting it would manufacture mirror_missing. + const canonicalProvider = await redis.hget?.(keys.canonical, "provider_id"); + if (canonicalProvider !== rollbackProviderValue && canonicalProvider !== undefined) { + await redis.eval( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + keys.legacyProvider, + rollbackProviderValue + ); + } } catch (error) { logger.warn("Legacy binding rollback could not execute atomically", { error: error instanceof Error ? error.message : String(error), diff --git a/tests/unit/lib/redis/session-binding.test.ts b/tests/unit/lib/redis/session-binding.test.ts index 58fc28c54..0bee1de34 100644 --- a/tests/unit/lib/redis/session-binding.test.ts +++ b/tests/unit/lib/redis/session-binding.test.ts @@ -897,6 +897,55 @@ describe("versioned session binding operations", () => { ); }); + it("keeps a provider mirror when canonical imported the same value before rollback", async () => { + let provider: string | null = null; + const mock = createMockRedis({ + operationResponses: { + [DELETE_LEGACY_PROVIDER_IF_VALUE]: [ + () => { + provider = null; + return 1; + }, + ], + }, + }); + let existsCalls = 0; + mock.existsMock.mockImplementation(async () => { + existsCalls += 1; + return existsCalls === 3 ? 1 : 0; + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return "4"; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.setexMock.mockImplementation(async (key: string, _ttl: number, value: string) => { + if (key === "session:sid:provider") provider = value; + return "OK"; + }); + mock.hgetMock.mockResolvedValue("10"); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "set", providerId: 10 }, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "canonical_exists", + legacyFallbackAllowed: false, + }); + expect(provider).toBe("10"); + expect(mock.evalMock).not.toHaveBeenCalledWith( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + "session:sid:provider", + "10" + ); + }); + it("does not restore a mirror when the concurrent canonical binding is a null tombstone", async () => { let provider: string | null = "8"; const mock = createMockRedis(); From f98a5582dd52fb51e7f32c2309ccda70a8c9dba1 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:47:46 -0400 Subject: [PATCH 35/89] fix(discovery): preserve rectifier and sticky binding semantics --- src/app/v1/_lib/proxy/forwarder.ts | 133 +++++++++++++++++++++++++---- 1 file changed, 117 insertions(+), 16 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 864d43313..3f9c362dd 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -4853,6 +4853,7 @@ export class ProxyForwarder { const stickySlaMs = Math.max(1, settings.stickySlaMs ?? 20_000); const totalTimeoutMs = Math.max(stickySlaMs, settings.racingTotalTimeoutMs ?? 60_000); const protocol = ProxyForwarder.discoveryProtocol(session); + const rawCrossProviderFallbackEnabled = session.isRawCrossProviderFallbackEnabled(); const coordinator = new DiscoveryCoordinator({ concurrency, maxRounds }); const bindingKeyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; // Provider selection normally populates this snapshot for a reused Sticky. @@ -4910,6 +4911,14 @@ export class ProxyForwarder { } }; + const getAttemptModelRedirect = (attempt: (typeof winner & { id: string }) | null) => { + if (!attempt) return undefined; + if (attempt.modelRedirect !== undefined) return attempt.modelRedirect; + const redirect = attempt.session.getCurrentModelRedirect(attempt.provider.id); + if (redirect) attempt.modelRedirect = structuredClone(redirect); + return attempt.modelRedirect; + }; + const cancelAttempt = (attempt: (typeof winner & { id: string }) | null, reason: string) => { if (attempt?.readerTransferred) return; if (!attempt?.pending) return; @@ -5046,7 +5055,15 @@ export class ProxyForwarder { }, delayMs); }; - const launch = async (provider: Provider, kind: "normal" | "fallback"): Promise => { + const launch = async ( + provider: Provider, + kind: "normal" | "fallback", + options?: { + attemptSession?: ProxySession; + requestAttemptCount?: number; + retryState?: ReactiveRectifierRetryState; + } + ): Promise => { if (settled || committed || launched.has(provider.id)) return; launched.add(provider.id); let providerSessionRefTracked = false; @@ -5083,9 +5100,10 @@ export class ProxyForwarder { let attemptSession: ProxySession; try { attemptSession = - provider.id === initialProvider.id + options?.attemptSession ?? + (provider.id === initialProvider.id ? session - : ProxyForwarder.createStreamingShadowSession(session, provider); + : ProxyForwarder.createStreamingShadowSession(session, provider)); attemptSession.setProvider(provider); } catch (error) { rollbackLaunch(); @@ -5112,8 +5130,8 @@ export class ProxyForwarder { clearResponseTimeout: null, firstByteTimeoutMs: 0, sequence: ++sequence, - requestAttemptCount: 1, - reactiveRectifierRetryState: { + requestAttemptCount: options?.requestAttemptCount ?? 1, + reactiveRectifierRetryState: options?.retryState ?? { thinkingSignatureRetried: false, thinkingBudgetRetried: false, thinkingEffortConflictRetried: false, @@ -5206,16 +5224,73 @@ export class ProxyForwarder { }) .catch(async (error) => { if (committed || settled || !attempt.pending) return; - attempt.pending = false; - const failureAction = coordinator.markFailed(id); lastError = error instanceof Error ? error : new Error(String(error)); lastErrorCategory = await categorizeErrorAsync(lastError); + const errorMessage = + lastError instanceof ProxyError + ? lastError.getDetailedErrorMessage() + : lastError.message; + + // Preserve the existing provider-local rectifier contract before + // classifying a 400 as terminal. The rectifier mutates the shadow + // request session, so retry the same attempt session rather than + // creating a fresh unrectified shadow from the parent session. + const rectifier = await tryApplyReactiveRectifier({ + provider, + requestSession: attempt.session, + persistSession: session, + errorMessage, + attemptNumber: attempt.requestAttemptCount, + retryAttemptNumber: attempt.requestAttemptCount + 1, + retryState: attempt.reactiveRectifierRetryState, + }); + if (rectifier.matched && rectifier.applied) { + attempt.pending = false; + coordinator.removeAttempt(id); + const readerCancel = attempt.reader?.cancel("discovery_rectifier_retry"); + readerCancel?.catch(() => undefined); + attempt.releaseAgent?.(); + releaseProviderRef(provider.id); + session.addProviderToChain(provider, { + ...buildRetryFailedChainEntry( + provider, + attempt.endpointAudit, + attempt.requestAttemptCount, + lastError, + errorMessage, + rectifier.requestDetailsBeforeRectify, + rawCrossProviderFallbackEnabled + ), + modelRedirect: getAttemptModelRedirect(attempt), + }); + launched.delete(provider.id); + try { + await launch(provider, attempt.kind, { + attemptSession: attempt.session, + requestAttemptCount: attempt.requestAttemptCount + 1, + retryState: attempt.reactiveRectifierRetryState, + }); + } catch (retryLaunchError) { + lastError = + retryLaunchError instanceof Error + ? retryLaunchError + : new Error(String(retryLaunchError)); + lastErrorCategory = await categorizeErrorAsync(lastError); + await settleFailure( + ProxyForwarder.resolveHedgeTerminalError(lastError, lastErrorCategory) + ); + } + return; + } + + attempt.pending = false; + const failureAction = coordinator.markFailed(id); session.addProviderToChain(provider, { ...attempt.endpointAudit, reason: "retry_failed", attemptNumber: attempt.sequence, statusCode: lastError instanceof ProxyError ? lastError.statusCode : undefined, - errorMessage: lastError.message, + errorMessage, }); if ( lastErrorCategory === ErrorCategory.PROVIDER_ERROR && @@ -5365,7 +5440,10 @@ export class ProxyForwarder { }, totalTimeoutMs); try { - const hasSticky = session.shouldReuseProvider() && !!session.sessionId; + const hasSticky = + session.shouldReuseProvider() && + !!session.sessionId && + bindingSnapshot?.providerId === initialProvider.id; try { await launch(initialProvider, "normal"); } catch (error) { @@ -5399,13 +5477,36 @@ export class ProxyForwarder { sticky.kind = "fallback"; coordinator.promoteToFallback(sticky.id); if (bindingSnapshot && bindingSnapshot.providerId === initialProvider.id) { - void SessionManager.clearVersionedSessionProvider( - bindingSnapshot, - initialProvider.id, - Math.ceil((settings.stickyTimeoutCooldownMs ?? 300_000) / 1000) - ).catch((error) => - logger.debug("[Discovery] Failed to clear timed-out Sticky", { error }) - ); + void (async () => { + const cleared = await SessionManager.clearVersionedSessionProvider( + bindingSnapshot, + initialProvider.id, + Math.ceil((settings.stickyTimeoutCooldownMs ?? 300_000) / 1000) + ); + if (cleared.status === "ok") { + bindingSnapshot = cleared.snapshot; + session.setSessionBindingSnapshot(cleared.snapshot); + } else { + logger.debug("[Discovery] Failed to clear timed-out Sticky", { + reason: cleared.reason, + }); + } + })() + .catch((error) => + logger.debug("[Discovery] Failed to clear timed-out Sticky", { error }) + ) + .finally(() => { + if (currentRound < maxRounds) { + void launchNextRound(Math.max(0, concurrency - 1)).catch((error) => + logger.warn("[Discovery] Sticky round launch failed", { error }) + ); + } else { + void onBoundary().catch((error) => + logger.warn("[Discovery] Sticky boundary failed", { error }) + ); + } + }); + return; } if (currentRound < maxRounds) { void launchNextRound(Math.max(0, concurrency - 1)).catch((error) => From 24a39ccdce21e97fe4407e88dbbe59c27d56a030 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:52:01 -0400 Subject: [PATCH 36/89] style(discovery): organize response handler imports --- src/app/v1/_lib/proxy/response-handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index 3b7ee3d7f..b1e13938d 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -13,9 +13,9 @@ import { requestCloudPriceTableSync } from "@/lib/price-sync/cloud-price-updater import { ProxyStatusTracker } from "@/lib/proxy-status-tracker"; import { RateLimitService } from "@/lib/rate-limit"; import { deleteLiveChain } from "@/lib/redis/live-chain-store"; +import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import { SessionManager } from "@/lib/session-manager"; import { SessionTracker } from "@/lib/session-tracker"; -import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import { CODEX_1M_CONTEXT_TOKEN_THRESHOLD } from "@/lib/special-attributes"; import type { CostBreakdown, From 8adf0207d5351a5411ccd1a11c6b342539768bb6 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:54:51 -0400 Subject: [PATCH 37/89] fix(binding): guard legacy owner refresh races --- src/lib/redis/session-binding.ts | 39 +++++++++++++- tests/unit/lib/redis/session-binding.test.ts | 55 ++++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/src/lib/redis/session-binding.ts b/src/lib/redis/session-binding.ts index 1076b4704..2048303ba 100644 --- a/src/lib/redis/session-binding.ts +++ b/src/lib/redis/session-binding.ts @@ -837,6 +837,20 @@ function parseLegacyProviderId(raw: string | null): number | null | undefined { return isPositiveInteger(providerId) ? providerId : undefined; } +async function ensureLegacyOwner( + redis: SessionBindingRedisClient, + keys: SessionBindingKeys, + keyId: number, + ttlSeconds: number +): Promise { + if ((await redis.expire(keys.legacyOwner, ttlSeconds)) > 0) return null; + + await redis.set(keys.legacyOwner, keyId.toString(), "EX", ttlSeconds, "NX"); + const owner = await redis.get(keys.legacyOwner); + if (owner === keyId.toString()) return null; + return conflict(owner === null ? "orphan_legacy_provider" : "foreign_legacy_owner"); +} + /** * A legacy fallback mutation can race a different worker which has already * recovered versioned binding capability. Re-check the canonical key after a @@ -960,6 +974,9 @@ export async function mutateLegacySessionBindingSafely( if (legacyProvider === undefined) return conflict("invalid_legacy_provider"); if ((await redis.exists(keys.canonical)) > 0) return conflict("canonical_exists"); + const ownerRefreshConflict = await ensureLegacyOwner(redis, keys, input.keyId, ttlSeconds); + if (ownerRefreshConflict) return ownerRefreshConflict; + switch (input.mutation.type) { case "inspect": return { status: "ok", changed: false, providerId: legacyProvider }; @@ -985,8 +1002,17 @@ export async function mutateLegacySessionBindingSafely( ttlSeconds, "NX" ); - await redis.expire(keys.legacyOwner, ttlSeconds); if (result === "OK") { + const ownerConflict = await ensureLegacyOwner(redis, keys, input.keyId, ttlSeconds); + if (ownerConflict) { + await redis.eval( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + keys.legacyProvider, + input.mutation.providerId.toString() + ); + return ownerConflict; + } const conflictAfterBind = await rejectLegacyMutationAfterCanonicalAppeared( redis, keys, @@ -1001,8 +1027,17 @@ export async function mutateLegacySessionBindingSafely( } case "set": await redis.setex(keys.legacyProvider, ttlSeconds, input.mutation.providerId.toString()); - await redis.expire(keys.legacyOwner, ttlSeconds); { + const ownerConflict = await ensureLegacyOwner(redis, keys, input.keyId, ttlSeconds); + if (ownerConflict) { + await redis.eval( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + keys.legacyProvider, + input.mutation.providerId.toString() + ); + return ownerConflict; + } const conflictAfterSet = await rejectLegacyMutationAfterCanonicalAppeared( redis, keys, diff --git a/tests/unit/lib/redis/session-binding.test.ts b/tests/unit/lib/redis/session-binding.test.ts index 0bee1de34..f51b4688b 100644 --- a/tests/unit/lib/redis/session-binding.test.ts +++ b/tests/unit/lib/redis/session-binding.test.ts @@ -946,6 +946,61 @@ describe("versioned session binding operations", () => { ); }); + it("rolls back a legacy provider bind when the owner cannot be refreshed", async () => { + let owner: string | null = "4"; + let provider: string | null = null; + const mock = createMockRedis({ + operationResponses: { + [DELETE_LEGACY_PROVIDER_IF_VALUE]: [ + () => { + provider = null; + return 1; + }, + ], + }, + }); + let expireCalls = 0; + mock.expireMock.mockImplementation(async () => { + expireCalls += 1; + if (expireCalls === 1) return 1; + owner = null; + return 0; + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return owner; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.setMock.mockImplementation(async () => { + owner = "5"; + return null; + }); + mock.setexMock.mockImplementation(async (key: string, _ttl: number, value: string) => { + if (key === "session:sid:provider") provider = value; + return "OK"; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "set", providerId: 10 }, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "foreign_legacy_owner", + legacyFallbackAllowed: false, + }); + expect(provider).toBeNull(); + expect(mock.evalMock).toHaveBeenCalledWith( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + "session:sid:provider", + "10" + ); + }); + it("does not restore a mirror when the concurrent canonical binding is a null tombstone", async () => { let provider: string | null = "8"; const mock = createMockRedis(); From a643ff8e231168f3ac0a39b972c3c9a978588bcb Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 16:59:11 -0400 Subject: [PATCH 38/89] fix(discovery): honor effective group priority --- src/app/v1/_lib/proxy/forwarder.ts | 2 +- src/app/v1/_lib/proxy/provider-selector.ts | 7 +++++++ .../provider-selector-group-priority.test.ts | 16 ++++++++++++++++ .../proxy-forwarder-hedge-first-byte.test.ts | 4 ++++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 91d78cdaf..120b15dd9 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -5119,7 +5119,7 @@ export class ProxyForwarder { coordinator.addAttempt({ id, providerId: provider.id, - priority: provider.priority || 0, + priority: ProxyProviderResolver.resolveEffectivePriorityForSession(provider, session), kind, ready: false, pending: true, diff --git a/src/app/v1/_lib/proxy/provider-selector.ts b/src/app/v1/_lib/proxy/provider-selector.ts index ee290e1aa..d4ce6fdbe 100644 --- a/src/app/v1/_lib/proxy/provider-selector.ts +++ b/src/app/v1/_lib/proxy/provider-selector.ts @@ -1179,6 +1179,13 @@ export class ProxyProviderResolver { return provider.priority ?? 0; } + static resolveEffectivePriorityForSession(provider: Provider, session: ProxySession): number { + return ProxyProviderResolver.resolveEffectivePriority( + provider, + getEffectiveProviderGroup(session) + ); + } + /** * 优先级分层：只选择最高优先级的供应商（支持分组优先级覆盖） */ diff --git a/tests/unit/proxy/provider-selector-group-priority.test.ts b/tests/unit/proxy/provider-selector-group-priority.test.ts index e0c6e6da5..143c851fa 100644 --- a/tests/unit/proxy/provider-selector-group-priority.test.ts +++ b/tests/unit/proxy/provider-selector-group-priority.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import type { ProxySession } from "@/app/v1/_lib/proxy/session"; import type { Provider } from "@/types/provider"; import { ProxyProviderResolver } from "@/app/v1/_lib/proxy/provider-selector"; @@ -128,6 +129,21 @@ describe("resolveEffectivePriority", () => { // "cli,admin" - only "cli" matches, should return 3 expect(ProxyProviderResolver.resolveEffectivePriority(provider, "cli,admin")).toBe(3); }); + + it("resolves Discovery priority from the authenticated key group", () => { + const provider = makeProvider({ + priority: 10, + groupPriorities: { cli: 1 }, + }); + const session = { + authState: { + key: { providerGroup: "cli" }, + user: { providerGroup: "chat" }, + }, + } as unknown as ProxySession; + + expect(ProxyProviderResolver.resolveEffectivePriorityForSession(provider, session)).toBe(1); + }); }); describe("selectTopPriority with group context", () => { diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index 1458d825d..5654e4745 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -4,6 +4,9 @@ import { resolveEndpointPolicy } from "@/app/v1/_lib/proxy/endpoint-policy"; const mocks = vi.hoisted(() => ({ pickRandomProviderWithExclusion: vi.fn(), pickDiscoveryProviders: vi.fn(), + resolveEffectivePriorityForSession: vi.fn( + (provider: { priority?: number | null }) => provider.priority ?? 0 + ), recordSuccess: vi.fn(), recordFailure: vi.fn(async () => {}), getCircuitState: vi.fn(() => "closed"), @@ -107,6 +110,7 @@ vi.mock("@/app/v1/_lib/proxy/provider-selector", () => ({ ProxyProviderResolver: { pickRandomProviderWithExclusion: mocks.pickRandomProviderWithExclusion, pickDiscoveryProviders: mocks.pickDiscoveryProviders, + resolveEffectivePriorityForSession: mocks.resolveEffectivePriorityForSession, }, })); From 21c1219037a06292a2bdd18aac5f39dc00e92d56 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 19:05:29 -0400 Subject: [PATCH 39/89] fix(binding): close legacy session races --- src/lib/redis/lua-scripts.ts | 44 +++ src/lib/redis/session-binding.ts | 301 +++++++++++++++++- src/lib/session-manager.ts | 57 +++- tests/unit/lib/redis/session-binding.test.ts | 271 +++++++++++++++- .../lib/session-manager-binding-smart.test.ts | 51 ++- .../session-manager-versioned-binding.test.ts | 55 ++++ 6 files changed, 762 insertions(+), 17 deletions(-) diff --git a/src/lib/redis/lua-scripts.ts b/src/lib/redis/lua-scripts.ts index 342cf35cb..4e3437dad 100644 --- a/src/lib/redis/lua-scripts.ts +++ b/src/lib/redis/lua-scripts.ts @@ -693,6 +693,50 @@ end return {'ok', 'cleared', next_generation, ''} `; +/** + * Renew a request-scoped Discovery lease only while the caller still owns it. + * + * KEYS[1]: tenant-scoped Discovery lease key + * ARGV[1]: owner token + * ARGV[2]: lease TTL in seconds + * + * Return: 1 when renewed, otherwise 0. + */ +export const RENEW_SESSION_DISCOVERY_LEASE = ` +local lease_key = KEYS[1] +local owner_token = ARGV[1] +local ttl = tonumber(ARGV[2]) + +if owner_token == '' or not ttl or ttl <= 0 then + return 0 +end +if redis.call('GET', lease_key) ~= owner_token then + return 0 +end + +return redis.call('EXPIRE', lease_key, ttl) +`; + +/** + * Release a request-scoped Discovery lease without deleting a newer owner's + * lease after the original owner's TTL elapsed. + * + * KEYS[1]: tenant-scoped Discovery lease key + * ARGV[1]: owner token + * + * Return: 1 when released, otherwise 0. + */ +export const RELEASE_SESSION_DISCOVERY_LEASE = ` +local lease_key = KEYS[1] +local owner_token = ARGV[1] + +if owner_token == '' or redis.call('GET', lease_key) ~= owner_token then + return 0 +end + +return redis.call('DEL', lease_key) +`; + /** * Tenant-authorized administrative termination. Unlike request-level clear, * this operation intentionally does not compare an old generation. It still diff --git a/src/lib/redis/session-binding.ts b/src/lib/redis/session-binding.ts index 2048303ba..41713d0f1 100644 --- a/src/lib/redis/session-binding.ts +++ b/src/lib/redis/session-binding.ts @@ -8,6 +8,8 @@ import { CLEAR_SESSION_BINDING, DELETE_LEGACY_PROVIDER_IF_VALUE, READ_OR_RECONCILE_SESSION_BINDING, + RELEASE_SESSION_DISCOVERY_LEASE, + RENEW_SESSION_DISCOVERY_LEASE, RESTORE_LEGACY_PROVIDER_IF_ABSENT, TERMINATE_SESSION_BINDING, } from "./lua-scripts"; @@ -127,6 +129,50 @@ export interface SessionProviderCooldownInput { redis?: SessionBindingRedisClient; } +export interface SessionDiscoveryLeaseInput { + sessionId: string; + keyId: number; + ownerToken: string; + redis?: SessionBindingRedisClient; +} + +export interface AcquireSessionDiscoveryLeaseInput { + sessionId: string; + keyId: number; + ttlSeconds: number; + ownerToken?: string; + redis?: SessionBindingRedisClient; +} + +export interface RenewSessionDiscoveryLeaseInput extends SessionDiscoveryLeaseInput { + ttlSeconds: number; +} + +export type SessionDiscoveryLeaseAcquireResult = + | { + status: "acquired"; + ownerToken: string; + legacyFallbackAllowed: false; + } + | { + status: "conflict"; + reason: "invalid_input" | "lease_held"; + legacyFallbackAllowed: false; + } + | SessionBindingUnavailableResult; + +export type SessionDiscoveryLeaseMutationResult = + | { + status: "renewed" | "released"; + legacyFallbackAllowed: false; + } + | { + status: "lost"; + reason: "invalid_input" | "not_owner_or_missing"; + legacyFallbackAllowed: false; + } + | SessionBindingUnavailableResult; + export interface SessionBindingOkResult { status: "ok"; snapshot: SessionBindingSnapshot; @@ -238,6 +284,17 @@ export function buildSessionProviderCooldownKey( ); } +export function buildSessionDiscoveryLeaseKey( + sessionId: string, + keyId: number, + namespace?: string +): string { + return namespacedKey( + namespace, + `session-binding:v1:{${bindingHashTag(sessionId, keyId)}}:discovery-lease` + ); +} + export function buildSessionBindingKeys( sessionId: string, keyId: number, @@ -289,6 +346,13 @@ function normalizeEvalResult(raw: unknown): string[] { }); } +function parseLeaseMutationFlag(raw: unknown): boolean { + const value = Buffer.isBuffer(raw) ? raw.toString("utf8") : raw; + if (value === 1 || value === "1") return true; + if (value === 0 || value === "0" || value === null) return false; + throw new Error("Invalid session Discovery lease Lua result"); +} + function parseProviderId(raw: string): number | null { if (raw === "") return null; const providerId = Number(raw); @@ -456,6 +520,21 @@ function handleOperationError(ready: ReadyClient, error: unknown): SessionBindin return unavailable("operation_failed"); } +function handleLeaseOperationError( + ready: ReadyClient, + error: unknown +): SessionBindingUnavailableResult { + if (isCapabilityError(error)) { + markCapabilityUnavailable(ready.redis, ready.epoch, error); + return unavailable("capability_unavailable"); + } + + logger.warn("Session Discovery lease operation failed", { + error: error instanceof Error ? error.message : String(error), + }); + return unavailable("operation_failed"); +} + async function runCapabilityProbe( client: SessionBindingRedisClient, epoch: number @@ -468,10 +547,18 @@ async function runCapabilityProbe( const ttlSeconds = 60; const keys = buildSessionBindingKeys(sessionId, keyId, namespace); const cooldownKey = buildSessionProviderCooldownKey(sessionId, keyId, providerId, namespace); - const cleanupKeys = [keys.canonical, keys.legacyProvider, keys.legacyOwner, cooldownKey]; + const leaseKey = buildSessionDiscoveryLeaseKey(sessionId, keyId, namespace); + const cleanupKeys = [ + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + cooldownKey, + leaseKey, + ]; const initialGeneration = randomUUID(); const boundGeneration = randomUUID(); const clearedGeneration = randomUUID(); + const leaseOwnerToken = randomUUID(); let operationsSucceeded = false; let cleanupSucceeded = false; @@ -553,6 +640,40 @@ async function runCapabilityProbe( if (cooldownGeneration !== cleared.snapshot.generation) { throw new Error("Session binding capability probe cooldown failed"); } + + const acquiredLease = await withCapabilityProbeDeadline( + client.set(leaseKey, leaseOwnerToken, "EX", ttlSeconds, "NX"), + deadlineAt + ); + if (acquiredLease !== "OK") { + throw new Error("Session binding capability probe lease acquire failed"); + } + + const renewedLease = parseLeaseMutationFlag( + await withCapabilityProbeDeadline( + client.eval( + RENEW_SESSION_DISCOVERY_LEASE, + 1, + leaseKey, + leaseOwnerToken, + ttlSeconds.toString() + ), + deadlineAt + ) + ); + if (!renewedLease) { + throw new Error("Session binding capability probe lease renew failed"); + } + + const releasedLease = parseLeaseMutationFlag( + await withCapabilityProbeDeadline( + client.eval(RELEASE_SESSION_DISCOVERY_LEASE, 1, leaseKey, leaseOwnerToken), + deadlineAt + ) + ); + if (!releasedLease) { + throw new Error("Session binding capability probe lease release failed"); + } operationsSucceeded = capabilityClient === client && capabilityEpoch === epoch; } catch (error) { logger.warn("Versioned session binding Redis capability probe failed", { @@ -656,6 +777,116 @@ function connectionIsCurrent(ready: ReadyClient): boolean { return capabilityClient === ready.redis && capabilityEpoch === ready.epoch; } +export async function acquireSessionDiscoveryLease( + input: AcquireSessionDiscoveryLeaseInput +): Promise { + const ownerToken = input.ownerToken ?? randomUUID(); + if (!isValidIdentity(input.sessionId, input.keyId, input.ttlSeconds) || ownerToken.length === 0) { + return { + status: "conflict", + reason: "invalid_input", + legacyFallbackAllowed: false, + }; + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + try { + const result = await ready.redis.set( + buildSessionDiscoveryLeaseKey(input.sessionId, input.keyId), + ownerToken, + "EX", + input.ttlSeconds, + "NX" + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + if (result === "OK" || (Buffer.isBuffer(result) && result.toString("utf8") === "OK")) { + return { status: "acquired", ownerToken, legacyFallbackAllowed: false }; + } + if (result === null) { + return { status: "conflict", reason: "lease_held", legacyFallbackAllowed: false }; + } + return handleLeaseOperationError(ready, new Error("Unexpected Discovery lease SET result")); + } catch (error) { + return handleLeaseOperationError(ready, error); + } +} + +export async function renewSessionDiscoveryLease( + input: RenewSessionDiscoveryLeaseInput +): Promise { + if ( + !isValidIdentity(input.sessionId, input.keyId, input.ttlSeconds) || + input.ownerToken.length === 0 + ) { + return { status: "lost", reason: "invalid_input", legacyFallbackAllowed: false }; + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + try { + const renewed = parseLeaseMutationFlag( + await evalBindingScript( + ready.redis, + RENEW_SESSION_DISCOVERY_LEASE, + 1, + buildSessionDiscoveryLeaseKey(input.sessionId, input.keyId), + input.ownerToken, + input.ttlSeconds.toString() + ) + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return renewed + ? { status: "renewed", legacyFallbackAllowed: false } + : { + status: "lost", + reason: "not_owner_or_missing", + legacyFallbackAllowed: false, + }; + } catch (error) { + return handleLeaseOperationError(ready, error); + } +} + +export async function releaseSessionDiscoveryLease( + input: SessionDiscoveryLeaseInput +): Promise { + if ( + input.sessionId.length === 0 || + !isPositiveInteger(input.keyId) || + input.ownerToken.length === 0 + ) { + return { status: "lost", reason: "invalid_input", legacyFallbackAllowed: false }; + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + try { + const released = parseLeaseMutationFlag( + await evalBindingScript( + ready.redis, + RELEASE_SESSION_DISCOVERY_LEASE, + 1, + buildSessionDiscoveryLeaseKey(input.sessionId, input.keyId), + input.ownerToken + ) + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return released + ? { status: "released", legacyFallbackAllowed: false } + : { + status: "lost", + reason: "not_owner_or_missing", + legacyFallbackAllowed: false, + }; + } catch (error) { + return handleLeaseOperationError(ready, error); + } +} + export async function readOrReconcileSessionBinding( input: ReadOrReconcileSessionBindingInput ): Promise { @@ -837,6 +1068,23 @@ function parseLegacyProviderId(raw: string | null): number | null | undefined { return isPositiveInteger(providerId) ? providerId : undefined; } +async function deleteLegacyProviderIfValue( + redis: SessionBindingRedisClient, + providerKey: string, + providerId: number +): Promise { + const result = await redis.eval( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + providerKey, + providerId.toString() + ); + const normalized = Buffer.isBuffer(result) ? result.toString("utf8") : result; + if (normalized === 1 || normalized === "1") return true; + if (normalized === 0 || normalized === "0" || normalized === null) return false; + throw new Error("Invalid conditional legacy provider delete result"); +} + async function ensureLegacyOwner( redis: SessionBindingRedisClient, keys: SessionBindingKeys, @@ -1059,7 +1307,26 @@ export async function mutateLegacySessionBindingSafely( if (legacyProvider === null) { return { status: "ok", changed: false, providerId: null }; } - await redis.del(keys.legacyProvider); + { + const deleted = await deleteLegacyProviderIfValue( + redis, + keys.legacyProvider, + legacyProvider + ); + if (!deleted) { + const conflictAfterConcurrentMutation = + await rejectLegacyMutationAfterCanonicalAppeared(redis, keys); + if (conflictAfterConcurrentMutation) return conflictAfterConcurrentMutation; + + const concurrentProvider = parseLegacyProviderId(await redis.get(keys.legacyProvider)); + if (concurrentProvider === undefined) return conflict("invalid_legacy_provider"); + if (concurrentProvider === null) { + await redis.expire(keys.legacyOwner, ttlSeconds); + return { status: "ok", changed: false, providerId: null }; + } + return conflict("provider_mismatch"); + } + } await redis.expire(keys.legacyOwner, ttlSeconds); { const conflictAfterClear = await rejectLegacyMutationAfterCanonicalAppeared( @@ -1078,6 +1345,36 @@ export async function mutateLegacySessionBindingSafely( ) { return conflict("provider_mismatch"); } + if (input.mutation.expectedProviderIds) { + const providerToTerminate = legacyProvider as number; + const deleted = await deleteLegacyProviderIfValue( + redis, + keys.legacyProvider, + providerToTerminate + ); + if (!deleted) { + const conflictAfterConcurrentMutation = + await rejectLegacyMutationAfterCanonicalAppeared(redis, keys); + if (conflictAfterConcurrentMutation) return conflictAfterConcurrentMutation; + return conflict("provider_mismatch"); + } + + // Provider-scoped invalidation must preserve tenant ownership. A + // concurrent failover may install a new Provider immediately after + // the conditional delete; removing the owner here would orphan it. + const ownerConflict = await ensureLegacyOwner(redis, keys, input.keyId, ttlSeconds); + if (ownerConflict) return ownerConflict; + + const conflictAfterTerminate = await rejectLegacyMutationAfterCanonicalAppeared( + redis, + keys, + undefined, + { value: providerToTerminate.toString(), ttlSeconds } + ); + if (conflictAfterTerminate) return conflictAfterTerminate; + return { status: "ok", changed: true, providerId: null }; + } + if (legacyProvider !== null) await redis.del(keys.legacyProvider); await redis.del(keys.legacyOwner); { diff --git a/src/lib/session-manager.ts b/src/lib/session-manager.ts index 02291f1c6..d57a3a38e 100644 --- a/src/lib/session-manager.ts +++ b/src/lib/session-manager.ts @@ -32,15 +32,21 @@ import { getUserActiveSessionsKey, } from "./redis/active-session-keys"; import { + acquireSessionDiscoveryLease as acquireVersionedSessionDiscoveryLease, clearSessionBinding as clearVersionedSessionBinding, compareAndSetSessionBinding, + ensureVersionedBindingCapability, mutateLegacySessionBindingSafely, readOrReconcileSessionBinding, isSessionProviderCoolingDown as readSessionProviderCooldown, getVersionedBindingCapabilityState as readVersionedBindingCapabilityState, + releaseSessionDiscoveryLease as releaseVersionedSessionDiscoveryLease, + renewSessionDiscoveryLease as renewVersionedSessionDiscoveryLease, type SessionBindingResult, type SessionBindingSnapshot, type SessionBindingUnavailableResult, + type SessionDiscoveryLeaseAcquireResult, + type SessionDiscoveryLeaseMutationResult, type SessionProviderCooldownResult, terminateSessionBinding as terminateVersionedSessionBinding, type VersionedBindingCapabilityState, @@ -668,6 +674,54 @@ export class SessionManager { return readVersionedBindingCapabilityState(); } + static async ensureVersionedBindingCapability(): Promise { + return ensureVersionedBindingCapability(); + } + + static async acquireSessionDiscoveryLease( + sessionId: string, + keyId: number, + ttlSeconds: number, + ownerToken?: string + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return acquireVersionedSessionDiscoveryLease({ + sessionId, + keyId, + ttlSeconds, + ownerToken, + redis, + }); + } + + static async renewSessionDiscoveryLease( + sessionId: string, + keyId: number, + ownerToken: string, + ttlSeconds: number + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return renewVersionedSessionDiscoveryLease({ + sessionId, + keyId, + ownerToken, + ttlSeconds, + redis, + }); + } + + static async releaseSessionDiscoveryLease( + sessionId: string, + keyId: number, + ownerToken: string + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return releaseVersionedSessionDiscoveryLease({ sessionId, keyId, ownerToken, redis }); + } + static async getSessionBindingSnapshot( sessionId: string, keyId: number @@ -1036,7 +1090,8 @@ export class SessionManager { /** * 智能更新 Session 绑定 * - * 策略：首次绑定用 SET NX；故障转移成功或竞速赢家强制改绑时无条件更新；其他情况按优先级和熔断状态决策 + * 策略：首次绑定用条件创建；故障转移成功或竞速赢家跳过优先级/熔断决策， + * 但版本化路径仍以读取到的 generation 做 CAS，避免迟到请求覆盖更新的绑定。 */ static async updateSessionBindingSmart( sessionId: string, diff --git a/tests/unit/lib/redis/session-binding.test.ts b/tests/unit/lib/redis/session-binding.test.ts index f51b4688b..7e235ab17 100644 --- a/tests/unit/lib/redis/session-binding.test.ts +++ b/tests/unit/lib/redis/session-binding.test.ts @@ -6,10 +6,12 @@ vi.mock("@/lib/logger", () => ({ })); import { + acquireSessionDiscoveryLease, buildCanonicalSessionBindingKey, buildLegacySessionOwnerKey, buildLegacySessionProviderKey, buildSessionBindingKeys, + buildSessionDiscoveryLeaseKey, buildSessionProviderCooldownKey, clearSessionBinding, compareAndSetSessionBinding, @@ -19,6 +21,8 @@ import { mutateLegacySessionBindingSafely, readOrReconcileSessionBinding, refreshSessionBinding, + releaseSessionDiscoveryLease, + renewSessionDiscoveryLease, resetVersionedBindingCapabilityForTests, terminateSessionBinding, type SessionBindingRedisClient, @@ -28,6 +32,8 @@ import { CLEAR_SESSION_BINDING, DELETE_LEGACY_PROVIDER_IF_VALUE, READ_OR_RECONCILE_SESSION_BINDING, + RELEASE_SESSION_DISCOVERY_LEASE, + RENEW_SESSION_DISCOVERY_LEASE, RESTORE_LEGACY_PROVIDER_IF_ABSENT, TERMINATE_SESSION_BINDING, } from "@/lib/redis/lua-scripts"; @@ -43,6 +49,7 @@ interface MockRedisOptions { probeGate?: Promise; status?: string; cooldownValue?: string | null; + leaseSetResult?: unknown; } function createMockRedis(options: MockRedisOptions = {}) { @@ -70,6 +77,8 @@ function createMockRedis(options: MockRedisOptions = {}) { probeCooldowns.set(String(args[5]), String(args[8])); return ["ok", "cleared", String(args[8]), ""]; } + if (script === RENEW_SESSION_DISCOVERY_LEASE) return 1; + if (script === RELEASE_SESSION_DISCOVERY_LEASE) return 1; throw new Error("Unexpected probe script"); } @@ -78,6 +87,13 @@ function createMockRedis(options: MockRedisOptions = {}) { if (response instanceof Error) throw response; if (typeof response === "function") return response(args); if (response !== undefined) return response; + if (script === DELETE_LEGACY_PROVIDER_IF_VALUE) { + const providerKey = String(args[2]); + const expectedProvider = String(args[3]); + if ((await getMock(providerKey)) !== expectedProvider) return 0; + await delMock(providerKey); + return 1; + } throw new Error(`Missing operation response for ${numberOfKeys} key script`); }); @@ -92,7 +108,10 @@ function createMockRedis(options: MockRedisOptions = {}) { }); const existsMock = vi.fn(async () => 0); const expireMock = vi.fn(async () => 1); - const setMock = vi.fn(async () => "OK"); + const setMock = vi.fn(async (key: string) => { + if (key.startsWith("session-binding-capability-probe:")) return "OK"; + return options.leaseSetResult === undefined ? "OK" : options.leaseSetResult; + }); const setexMock = vi.fn(async () => "OK"); const evalShaMock = vi.fn(async (..._args: unknown[]) => { if (options.evalShaNoScriptOnce) { @@ -161,13 +180,19 @@ describe("session binding key builders", () => { it("scopes canonical and cooldown keys by API key while preserving legacy mirrors", () => { const canonical = buildCanonicalSessionBindingKey("session:{a}", 17); const cooldown = buildSessionProviderCooldownKey("session:{a}", 17, 4); + const lease = buildSessionDiscoveryLeaseKey("session:{a}", 17); expect(canonical).toMatch(/^session-binding:v1:\{[a-f0-9]{64}\}:binding$/); expect(cooldown).toMatch(/^session-binding:v1:\{[a-f0-9]{64}\}:provider:4:cooldown$/); + expect(lease).toMatch(/^session-binding:v1:\{[a-f0-9]{64}\}:discovery-lease$/); expect(canonical.match(/\{([^}]+)\}/)?.[1]).toBe(cooldown.match(/\{([^}]+)\}/)?.[1]); + expect(canonical.match(/\{([^}]+)\}/)?.[1]).toBe(lease.match(/\{([^}]+)\}/)?.[1]); expect(buildCanonicalSessionBindingKey("session:a", 18)).not.toBe( buildCanonicalSessionBindingKey("session:a", 17) ); + expect(buildSessionDiscoveryLeaseKey("session:a", 18)).not.toBe( + buildSessionDiscoveryLeaseKey("session:a", 17) + ); expect(buildLegacySessionProviderKey("session:{a}")).toBe("session:session:{a}:provider"); expect(buildLegacySessionOwnerKey("session:{a}")).toBe("session:session:{a}:key"); }); @@ -192,9 +217,10 @@ describe("versioned binding capability", () => { await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("available"); expect(getVersionedBindingCapabilityState()).toBe("available"); - expect(mock.evalMock).toHaveBeenCalledTimes(3); + expect(mock.evalMock).toHaveBeenCalledTimes(5); expect(mock.getMock).toHaveBeenCalledTimes(1); - expect(mock.delMock).toHaveBeenCalledTimes(4); + expect(mock.setMock).toHaveBeenCalledTimes(1); + expect(mock.delMock).toHaveBeenCalledTimes(5); expect(mock.delMock.mock.calls.every((call) => call.length === 1)).toBe(true); expect(mock.onMock.mock.calls.map(([event]) => event)).toEqual([ "close", @@ -218,7 +244,7 @@ describe("versioned binding capability", () => { releaseProbe?.(); await expect(Promise.all([first, second])).resolves.toEqual(["available", "available"]); - expect(mock.evalMock).toHaveBeenCalledTimes(3); + expect(mock.evalMock).toHaveBeenCalledTimes(5); }); it("stays unavailable on the same connection after a capability failure", async () => { @@ -241,7 +267,7 @@ describe("versioned binding capability", () => { expect(getVersionedBindingCapabilityState()).toBe("unknown"); await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("available"); - expect(mock.evalMock).toHaveBeenCalledTimes(4); + expect(mock.evalMock).toHaveBeenCalledTimes(6); }); it("automatically probes when a reconnected client becomes ready", async () => { @@ -254,15 +280,15 @@ describe("versioned binding capability", () => { mock.emit("ready"); await vi.waitFor(() => expect(getVersionedBindingCapabilityState()).toBe("available")); - expect(mock.evalMock).toHaveBeenCalledTimes(4); + expect(mock.evalMock).toHaveBeenCalledTimes(6); }); it("does not become available when isolated probe cleanup fails", async () => { const mock = createMockRedis({ cleanupFails: true }); await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("unavailable"); - expect(mock.evalMock).toHaveBeenCalledTimes(3); - expect(mock.delMock).toHaveBeenCalledTimes(4); + expect(mock.evalMock).toHaveBeenCalledTimes(5); + expect(mock.delMock).toHaveBeenCalledTimes(5); }); it("detaches lifecycle listeners when tests reset state", async () => { @@ -287,6 +313,164 @@ describe("versioned binding capability", () => { }); }); +describe("session Discovery lease operations", () => { + beforeEach(() => { + resetVersionedBindingCapabilityForTests(); + }); + + it("acquires a tenant-scoped lease with an explicit owner token and TTL", async () => { + const mock = createMockRedis(); + + const result = await acquireSessionDiscoveryLease({ + sessionId: "sid", + keyId: 4, + ttlSeconds: 61, + ownerToken: "owner-a", + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "acquired", + ownerToken: "owner-a", + legacyFallbackAllowed: false, + }); + expect(mock.setMock.mock.calls.at(-1)).toEqual([ + buildSessionDiscoveryLeaseKey("sid", 4), + "owner-a", + "EX", + 61, + "NX", + ]); + }); + + it("returns a lease conflict without revealing the current owner", async () => { + const mock = createMockRedis({ leaseSetResult: null }); + + const result = await acquireSessionDiscoveryLease({ + sessionId: "sid", + keyId: 4, + ttlSeconds: 30, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "lease_held", + legacyFallbackAllowed: false, + }); + }); + + it("renews and releases only through owner-token Lua primitives", async () => { + const mock = createMockRedis({ + operationResponses: { + [RENEW_SESSION_DISCOVERY_LEASE]: [1], + [RELEASE_SESSION_DISCOVERY_LEASE]: [1], + }, + }); + + await expect( + renewSessionDiscoveryLease({ + sessionId: "sid", + keyId: 4, + ownerToken: "owner-a", + ttlSeconds: 45, + redis: mock.redis, + }) + ).resolves.toEqual({ status: "renewed", legacyFallbackAllowed: false }); + await expect( + releaseSessionDiscoveryLease({ + sessionId: "sid", + keyId: 4, + ownerToken: "owner-a", + redis: mock.redis, + }) + ).resolves.toEqual({ status: "released", legacyFallbackAllowed: false }); + + expect(mock.evalMock).toHaveBeenCalledWith( + RENEW_SESSION_DISCOVERY_LEASE, + 1, + buildSessionDiscoveryLeaseKey("sid", 4), + "owner-a", + "45" + ); + expect(mock.evalMock).toHaveBeenCalledWith( + RELEASE_SESSION_DISCOVERY_LEASE, + 1, + buildSessionDiscoveryLeaseKey("sid", 4), + "owner-a" + ); + }); + + it("reports a lost lease when renew or release no longer owns the key", async () => { + const mock = createMockRedis({ + operationResponses: { + [RENEW_SESSION_DISCOVERY_LEASE]: [0], + [RELEASE_SESSION_DISCOVERY_LEASE]: [0], + }, + }); + + await expect( + renewSessionDiscoveryLease({ + sessionId: "sid", + keyId: 4, + ownerToken: "stale-owner", + ttlSeconds: 45, + redis: mock.redis, + }) + ).resolves.toEqual({ + status: "lost", + reason: "not_owner_or_missing", + legacyFallbackAllowed: false, + }); + await expect( + releaseSessionDiscoveryLease({ + sessionId: "sid", + keyId: 4, + ownerToken: "stale-owner", + redis: mock.redis, + }) + ).resolves.toEqual({ + status: "lost", + reason: "not_owner_or_missing", + legacyFallbackAllowed: false, + }); + }); + + it("rejects invalid identities before touching Redis", async () => { + const mock = createMockRedis(); + + await expect( + acquireSessionDiscoveryLease({ + sessionId: "", + keyId: 0, + ttlSeconds: 0, + ownerToken: "", + redis: mock.redis, + }) + ).resolves.toEqual({ + status: "conflict", + reason: "invalid_input", + legacyFallbackAllowed: false, + }); + await expect( + renewSessionDiscoveryLease({ + sessionId: "sid", + keyId: 4, + ttlSeconds: 30, + ownerToken: "", + redis: mock.redis, + }) + ).resolves.toEqual({ + status: "lost", + reason: "invalid_input", + legacyFallbackAllowed: false, + }); + + expect(mock.evalMock).not.toHaveBeenCalled(); + expect(mock.setMock).not.toHaveBeenCalled(); + }); +}); + describe("versioned session binding operations", () => { beforeEach(() => { resetVersionedBindingCapabilityForTests(); @@ -662,7 +846,7 @@ describe("versioned session binding operations", () => { keyId: 4, redis: mock.redis, }); - await vi.waitFor(() => expect(mock.evalMock).toHaveBeenCalledTimes(4)); + await vi.waitFor(() => expect(mock.evalMock).toHaveBeenCalledTimes(6)); mock.emit("reconnecting"); resolveOperation?.(["ok", "existing", "generation", "8"]); @@ -1039,6 +1223,73 @@ describe("versioned session binding operations", () => { ); }); + it("does not clear a newer legacy Provider that replaced the expected value", async () => { + let provider: string | null = "8"; + const mock = createMockRedis({ + operationResponses: { + [DELETE_LEGACY_PROVIDER_IF_VALUE]: [ + () => { + provider = "9"; + return 0; + }, + ], + }, + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return "4"; + if (key === "session:sid:provider") return provider; + return null; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "clear", expectedProviderId: 8 }, + }); + + expect(result).toMatchObject({ status: "conflict", reason: "provider_mismatch" }); + expect(provider).toBe("9"); + expect(mock.delMock).not.toHaveBeenCalledWith("session:sid:provider"); + }); + + it("does not terminate a newer legacy Provider or its tenant owner", async () => { + let owner: string | null = "4"; + let provider: string | null = "8"; + const mock = createMockRedis({ + operationResponses: { + [DELETE_LEGACY_PROVIDER_IF_VALUE]: [ + () => { + provider = "9"; + return 0; + }, + ], + }, + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return owner; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.delMock.mockImplementation(async (key: string) => { + if (key === "session:sid:provider") provider = null; + if (key === "session:sid:key") owner = null; + return 1; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "terminate", expectedProviderIds: [8] }, + }); + + expect(result).toMatchObject({ status: "conflict", reason: "provider_mismatch" }); + expect(provider).toBe("9"); + expect(owner).toBe("4"); + expect(mock.delMock).not.toHaveBeenCalled(); + }); + it("uses the tenant-authorized termination primitive and leaves a tombstone", async () => { const mock = createMockRedis({ operationResponses: { @@ -1157,7 +1408,7 @@ describe("versioned session binding operations", () => { mutation: { type: "terminate", expectedProviderIds: [10] }, }) ).resolves.toMatchObject({ status: "ok", changed: true, providerId: null }); - expect(owner).toBeNull(); + expect(owner).toBe("4"); }); it("rejects malformed legacy mutation arguments before touching Redis", async () => { diff --git a/tests/unit/lib/session-manager-binding-smart.test.ts b/tests/unit/lib/session-manager-binding-smart.test.ts index 61fc72c95..b7d9600b6 100644 --- a/tests/unit/lib/session-manager-binding-smart.test.ts +++ b/tests/unit/lib/session-manager-binding-smart.test.ts @@ -3,9 +3,9 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; /** * Tests for SessionManager.updateSessionBindingSmart forceUpdate semantics. * - * Hedge race winners must unconditionally rebind the session-reuse binding to - * the winner. forceUpdate short-circuits the smart-decision path (priority / - * circuit health) that would otherwise keep a healthy higher-priority binding. + * Hedge race winners bypass the smart-decision path (priority / circuit + * health), while versioned bindings still use generation CAS so a stale winner + * cannot overwrite a newer concurrent binding. */ let redisClientRef: { @@ -180,7 +180,7 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { expect(findProviderById).not.toHaveBeenCalled(); expect(isCircuitOpen).not.toHaveBeenCalled(); - // forceUpdate goes straight to the unconditional pipeline path. + // forceUpdate goes straight to the persistence path. expect(findProviderById).not.toHaveBeenCalled(); }); @@ -282,6 +282,49 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { expect(redisClientRef!.setex).not.toHaveBeenCalled(); }); + it.each([ + { isFailoverSuccess: false, forceUpdate: true }, + { isFailoverSuccess: true, forceUpdate: false }, + ])( + "does not let a stale versioned winner overwrite a newer binding (%o)", + async ({ isFailoverSuccess, forceUpdate }) => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "ok", + source: "existing", + snapshot: { + sessionId: SID, + keyId: KEY_ID, + providerId: 1, + generation: "generation-before-concurrent-update", + }, + legacyFallbackAllowed: false, + }); + bindingMocks.compareAndSetSessionBinding.mockResolvedValue({ + status: "conflict", + reason: "generation_mismatch", + legacyFallbackAllowed: false, + }); + + const result = await SessionManager.updateSessionBindingSmart( + SID, + 2, + 10, + false, + isFailoverSuccess, + KEY_ID, + forceUpdate + ); + + expect(result).toEqual({ + updated: false, + reason: "concurrent_binding_changed", + details: "Session binding changed before the update committed", + }); + expect(bindingMocks.compareAndSetSessionBinding).toHaveBeenCalledTimes(1); + expect(bindingMocks.mutateLegacySessionBindingSafely).not.toHaveBeenCalled(); + } + ); + it("does not fall back to legacy writes for a foreign owner conflict", async () => { bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ status: "conflict", diff --git a/tests/unit/lib/session-manager-versioned-binding.test.ts b/tests/unit/lib/session-manager-versioned-binding.test.ts index 57d9f5252..7078aa0ef 100644 --- a/tests/unit/lib/session-manager-versioned-binding.test.ts +++ b/tests/unit/lib/session-manager-versioned-binding.test.ts @@ -1,11 +1,14 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const bindingMocks = vi.hoisted(() => ({ + acquireSessionDiscoveryLease: vi.fn(), clearSessionBinding: vi.fn(), compareAndSetSessionBinding: vi.fn(), isSessionProviderCoolingDown: vi.fn(), readOrReconcileSessionBinding: vi.fn(), refreshSessionBinding: vi.fn(), + releaseSessionDiscoveryLease: vi.fn(), + renewSessionDiscoveryLease: vi.fn(), })); let redisClientRef: { @@ -75,9 +78,61 @@ beforeEach(() => { setex: vi.fn(async () => "OK"), }; bindingMocks.readOrReconcileSessionBinding.mockResolvedValue(snapshot()); + bindingMocks.acquireSessionDiscoveryLease.mockResolvedValue({ + status: "acquired", + ownerToken: "owner-a", + legacyFallbackAllowed: false, + }); + bindingMocks.renewSessionDiscoveryLease.mockResolvedValue({ + status: "renewed", + legacyFallbackAllowed: false, + }); + bindingMocks.releaseSessionDiscoveryLease.mockResolvedValue({ + status: "released", + legacyFallbackAllowed: false, + }); }); describe("SessionManager versioned binding adapter", () => { + it("delegates the complete tenant-scoped Discovery lease lifecycle", async () => { + await expect( + SessionManager.acquireSessionDiscoveryLease(SESSION_ID, KEY_ID, 61, "owner-a") + ).resolves.toMatchObject({ status: "acquired", ownerToken: "owner-a" }); + await expect( + SessionManager.renewSessionDiscoveryLease(SESSION_ID, KEY_ID, "owner-a", 61) + ).resolves.toMatchObject({ status: "renewed" }); + await expect( + SessionManager.releaseSessionDiscoveryLease(SESSION_ID, KEY_ID, "owner-a") + ).resolves.toMatchObject({ status: "released" }); + + expect(bindingMocks.acquireSessionDiscoveryLease).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SESSION_ID, + keyId: KEY_ID, + ttlSeconds: 61, + ownerToken: "owner-a", + redis: redisClientRef, + }) + ); + expect(bindingMocks.renewSessionDiscoveryLease).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SESSION_ID, + keyId: KEY_ID, + ttlSeconds: 61, + ownerToken: "owner-a", + redis: redisClientRef, + }) + ); + expect(bindingMocks.releaseSessionDiscoveryLease).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SESSION_ID, + keyId: KEY_ID, + ownerToken: "owner-a", + redis: redisClientRef, + }) + ); + }); + it("returns the tenant-scoped provider without reading the legacy mirror", async () => { await expect(SessionManager.getSessionProvider(SESSION_ID, KEY_ID)).resolves.toBe(PROVIDER_ID); From e3cb570ca89c79bd70d76419815a5acb7b230a7a Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 19:06:56 -0400 Subject: [PATCH 40/89] fix(settings): tighten discovery configuration --- messages/zh-TW/settings/config.json | 4 +- .../_components/system-settings-form.tsx | 69 +++++++++++-------- src/lib/api/v1/schemas/system-config.ts | 2 +- src/lib/validation/schemas.ts | 2 +- .../system-settings-discovery.test.ts | 5 ++ 5 files changed, 48 insertions(+), 34 deletions(-) diff --git a/messages/zh-TW/settings/config.json b/messages/zh-TW/settings/config.json index 3e4a4b2dc..8ccc6318a 100644 --- a/messages/zh-TW/settings/config.json +++ b/messages/zh-TW/settings/config.json @@ -124,8 +124,8 @@ "discoveryEnabledDesc": "啟用後，冷啟動串流請求會在限定視窗內探測多個供應商，並且最多保留一個保底請求。預設關閉。", "discoveryConcurrency": "Discovery 首輪並發數", "maxDiscoveryRounds": "Discovery 最大輪數", - "discoverySlaMs": "Discovery SLA（毫秒）", - "stickySlaMs": "Sticky SLA（毫秒）", + "discoverySlaMs": "Discovery 每輪 SLA（毫秒）", + "stickySlaMs": "Sticky 等待 SLA（毫秒）", "racingTotalTimeoutMs": "Discovery 總逾時（毫秒）", "stickyTimeoutCooldownMs": "Sticky 逾時冷卻（毫秒）", "discoveryWindowDesc": "總逾時必須不小於 Sticky SLA + 最大輪數 × Discovery SLA。Discovery 輸家會取消，不走舊 Hedge 的 drain 或輸家計費。", diff --git a/src/app/[locale]/settings/config/_components/system-settings-form.tsx b/src/app/[locale]/settings/config/_components/system-settings-form.tsx index 2642cbf8a..c887a6d2f 100644 --- a/src/app/[locale]/settings/config/_components/system-settings-form.tsx +++ b/src/app/[locale]/settings/config/_components/system-settings-form.tsx @@ -719,37 +719,46 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) disabled={isPending} />
-
- {( - [ - ["discoveryConcurrency", discoveryConcurrency, setDiscoveryConcurrency, 1], - ["maxDiscoveryRounds", maxDiscoveryRounds, setMaxDiscoveryRounds, 1], - ["discoverySlaMs", discoverySlaMs, setDiscoverySlaMs, 1], - ["stickySlaMs", stickySlaMs, setStickySlaMs, 1], - ["racingTotalTimeoutMs", racingTotalTimeoutMs, setRacingTotalTimeoutMs, 1], - ["stickyTimeoutCooldownMs", stickyTimeoutCooldownMs, setStickyTimeoutCooldownMs, 1], - ] as const - ).map(([key, value, setter, min]) => ( -
- - - setter(event.target.value === "" ? "" : Number(event.target.value)) - } - disabled={isPending || !discoveryEnabled} - className={inputClassName} - /> + {discoveryEnabled ? ( + <> +
+ {( + [ + ["discoveryConcurrency", discoveryConcurrency, setDiscoveryConcurrency, 2], + ["maxDiscoveryRounds", maxDiscoveryRounds, setMaxDiscoveryRounds, 1], + ["discoverySlaMs", discoverySlaMs, setDiscoverySlaMs, 1], + ["stickySlaMs", stickySlaMs, setStickySlaMs, 1], + ["racingTotalTimeoutMs", racingTotalTimeoutMs, setRacingTotalTimeoutMs, 1], + [ + "stickyTimeoutCooldownMs", + stickyTimeoutCooldownMs, + setStickyTimeoutCooldownMs, + 1, + ], + ] as const + ).map(([key, value, setter, min]) => ( +
+ + + setter(event.target.value === "" ? "" : Number(event.target.value)) + } + disabled={isPending} + className={inputClassName} + /> +
+ ))}
- ))} -
-

{t("discoveryWindowDesc")}

+

{t("discoveryWindowDesc")}

+ + ) : null}
{/* Verbose Provider Error */} diff --git a/src/lib/api/v1/schemas/system-config.ts b/src/lib/api/v1/schemas/system-config.ts index cdd5d7a12..977014298 100644 --- a/src/lib/api/v1/schemas/system-config.ts +++ b/src/lib/api/v1/schemas/system-config.ts @@ -102,7 +102,7 @@ export const SystemSettingsSchema = z discoveryConcurrency: z .number() .int() - .positive() + .min(2) .describe("Maximum number of normal Discovery attempts in the initial batch."), maxDiscoveryRounds: z.number().int().positive().describe("Maximum number of Discovery rounds."), discoverySlaMs: z diff --git a/src/lib/validation/schemas.ts b/src/lib/validation/schemas.ts index f347fc7ba..66bd6741c 100644 --- a/src/lib/validation/schemas.ts +++ b/src/lib/validation/schemas.ts @@ -1011,7 +1011,7 @@ export const UpdateSystemSettingsSchema = z discoveryConcurrency: z.coerce .number() .int("Discovery 并发数必须是整数") - .min(1, "Discovery 并发数必须大于 0") + .min(2, "Discovery 并发数不能小于 2") .max(32, "Discovery 并发数不能超过 32") .optional(), maxDiscoveryRounds: z.coerce diff --git a/tests/unit/validation/system-settings-discovery.test.ts b/tests/unit/validation/system-settings-discovery.test.ts index 1950ef225..8c831ddcf 100644 --- a/tests/unit/validation/system-settings-discovery.test.ts +++ b/tests/unit/validation/system-settings-discovery.test.ts @@ -15,6 +15,11 @@ describe("UpdateSystemSettingsSchema Discovery settings", () => { expect(result.discoveryConcurrency).toBe(2); }); + it("requires at least one fallback slot in addition to the normal lane", () => { + const result = UpdateSystemSettingsSchema.safeParse({ discoveryConcurrency: 1 }); + expect(result.success).toBe(false); + }); + it("rejects a total deadline shorter than the configured discovery window", () => { expect(() => UpdateSystemSettingsSchema.parse({ From ca12624f53c701c1d7f5dd5cb4e5095fd2b3ee50 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 19:09:54 -0400 Subject: [PATCH 41/89] fix(discovery): harden bounded streaming lifecycle --- .env.example | 1 + .../v1/_lib/proxy/discovery-coordinator.ts | 56 +- src/app/v1/_lib/proxy/discovery-validity.ts | 41 +- src/app/v1/_lib/proxy/forwarder.ts | 1074 ++++++++++++---- src/app/v1/_lib/proxy/provider-selector.ts | 4 +- src/app/v1/_lib/proxy/response-handler.ts | 721 ++++++++--- src/app/v1/_lib/proxy/session.ts | 53 +- src/app/v1/_lib/proxy/stream-finalization.ts | 13 + src/lib/config/env.schema.ts | 4 + src/lib/observability/discovery-metrics.ts | 126 ++ .../integration/proxy-hedge-lifecycle.test.ts | 127 +- .../unit/proxy/discovery-coordinator.test.ts | 60 +- tests/unit/proxy/discovery-validity.test.ts | 42 + .../proxy-forwarder-hedge-first-byte.test.ts | 1098 +++++++++++++++++ ...forwarder-provider-session-release.test.ts | 14 + ...esponse-handler-client-abort-drain.test.ts | 257 ++++ ...handler-endpoint-circuit-isolation.test.ts | 761 ++++++++++++ ...gemini-stream-passthrough-timeouts.test.ts | 2 +- 18 files changed, 4051 insertions(+), 403 deletions(-) create mode 100644 src/lib/observability/discovery-metrics.ts diff --git a/.env.example b/.env.example index 19b417d51..653ce134c 100644 --- a/.env.example +++ b/.env.example @@ -97,6 +97,7 @@ ENABLE_API_KEY_REDIS_CACHE="true" # 是否启用 API Key Redis 缓存（ # 降低该值会按签发时间收紧已签发 ADMIN_TOKEN 签名 cookie 的剩余寿命，且不会延长其原始 exp。 AUTH_SESSION_TTL_SECONDS=604800 # Web UI 登录态过期时间（秒，默认 604800 = 7 天，范围 60-31536000） SESSION_TTL=300 # 代理请求上下文缓存时间（秒，默认 300 = 5 分钟；不控制 Web UI 登录态） +DISCOVERY_ROLLOUT_PERCENT=100 # Discovery 运维灰度比例（0-100，按 API Key + Session 稳定分桶） STORE_SESSION_MESSAGES=false # 会话消息存储模式（默认：false） # - false：存储请求/响应体但对 message 内容脱敏 [REDACTED] # - true：原样存储 message 内容（注意隐私和存储空间影响） diff --git a/src/app/v1/_lib/proxy/discovery-coordinator.ts b/src/app/v1/_lib/proxy/discovery-coordinator.ts index 4a94baf70..f1330df92 100644 --- a/src/app/v1/_lib/proxy/discovery-coordinator.ts +++ b/src/app/v1/_lib/proxy/discovery-coordinator.ts @@ -66,7 +66,21 @@ export class DiscoveryCoordinator { return { requestEpoch: this.requestEpoch, roundEpoch: this.roundEpoch }; } + startStickyProbe(): void { + if (!this.isTerminal) this.state = "STICKY_PROBING"; + } + + startDiscoveryAfterSticky(): void { + if (this.state === "STICKY_PROBING" || this.state === "FALLBACK_READY_HELD") { + this.state = "DISCOVERY_RACING"; + } + } + beginRound(): { requestEpoch: number; roundEpoch: number; round: number } { + if (this.isTerminal || this.round >= this.maxRounds) { + return { ...this.epochs, round: this.round }; + } + this.round += 1; this.roundEpoch += 1; if (!this.isTerminal) this.state = "DISCOVERY_RACING"; return { ...this.epochs, round: this.round }; @@ -82,8 +96,22 @@ export class DiscoveryCoordinator { this.attempts.delete(id); } + /** Mark an already-running attempt as the sole fallback for this request. */ + promoteToFallback(id: string): boolean { + if (this.isTerminal) return false; + const attempt = this.attempts.get(id); + if (!attempt?.pending) return false; + attempt.kind = "fallback"; + this.state = "FALLBACK_READY_HELD"; + return true; + } + get isTerminal(): boolean { - return this.state === "WINNER_COMMITTED" || this.state === "TERMINAL_FAILED"; + return ( + this.state === "WINNER_COMMITTED" || + this.state === "FALLBACK_ACTIVE" || + this.state === "TERMINAL_FAILED" + ); } get activeAttempts(): DiscoveryAttempt[] { @@ -143,9 +171,11 @@ export class DiscoveryCoordinator { ): DiscoveryAction { if (!this.acceptsEpoch(requestEpoch, roundEpoch) || this.isTerminal) return { type: "none" }; const attempt = this.attempts.get(id); - if (!attempt) return { type: "none" }; + if (!attempt?.pending) return { type: "none" }; attempt.pending = false; attempt.ready = false; + const readyAction = this.chooseReadyNormal(); + if (readyAction.type !== "none") return readyAction; return this.afterAttemptState(); } @@ -166,8 +196,7 @@ export class DiscoveryCoordinator { ); if (higherTierPending) return { type: "none" }; } - const sameTier = readyNormal.filter((attempt) => attempt.priority === bestPriority); - const winner = sameTier[0]; + const winner = readyNormal[0]; this.state = "WINNER_COMMITTED"; winner.pending = false; return { @@ -202,12 +231,11 @@ export class DiscoveryCoordinator { const cancelAttemptIds = pendingNormal.map((attempt) => attempt.id); for (const attempt of pendingNormal) attempt.pending = false; if (this.round < this.maxRounds) { - this.round += 1; - this.roundEpoch += 1; + this.beginRound(); this.state = "DISCOVERY_RACING"; return { type: "launch", - slots: Math.max(1, this.concurrency - 1), + slots: Math.max(0, this.concurrency - 1), cancelAttemptIds, }; } @@ -228,12 +256,11 @@ export class DiscoveryCoordinator { for (const id of losers) this.attempts.get(id)!.pending = false; if (this.round < this.maxRounds) { - this.round += 1; - this.roundEpoch += 1; + this.beginRound(); this.state = "DISCOVERY_RACING"; return { type: "launch", - slots: Math.max(1, this.concurrency - 1), + slots: Math.max(0, this.concurrency - 1), cancelAttemptIds: losers, promoteAttemptId: fallback.id, }; @@ -243,6 +270,8 @@ export class DiscoveryCoordinator { onDeadline(): DiscoveryAction { if (this.isTerminal) return { type: "none" }; + const readyNormal = this.chooseReadyNormal(true); + if (readyNormal.type === "commit_normal") return readyNormal; const fallback = Array.from(this.attempts.values()).find( (attempt) => attempt.pending && attempt.kind === "fallback" && attempt.ready ); @@ -259,7 +288,7 @@ export class DiscoveryCoordinator { const attempt = this.attempts.get(id); if (!attempt || this.isTerminal) return { type: "none" }; attempt.pending = false; - this.state = "WINNER_COMMITTED"; + this.state = attempt.kind === "fallback" ? "FALLBACK_ACTIVE" : "WINNER_COMMITTED"; return { type: attempt.kind === "fallback" ? "promote_fallback" : "commit_normal", attemptId: id, @@ -297,9 +326,8 @@ export class DiscoveryCoordinator { this.state = "TERMINAL_FAILED"; return { type: "terminal_failure" }; } - this.round += 1; - this.roundEpoch += 1; + this.beginRound(); this.state = "DISCOVERY_RACING"; - return { type: "launch", slots: Math.max(1, this.concurrency - 1) }; + return { type: "launch", slots: this.concurrency }; } } diff --git a/src/app/v1/_lib/proxy/discovery-validity.ts b/src/app/v1/_lib/proxy/discovery-validity.ts index 198d77699..3325295a7 100644 --- a/src/app/v1/_lib/proxy/discovery-validity.ts +++ b/src/app/v1/_lib/proxy/discovery-validity.ts @@ -9,8 +9,12 @@ export type DiscoveryValidity = { ready: boolean; terminal: boolean; error: boolean; + limitExceeded?: boolean; }; +export const DISCOVERY_PREFIX_MAX_BYTES = 1024 * 1024; +export const DISCOVERY_EVENT_MAX_COUNT = 1024; + function hasContent(value: unknown): boolean { if (typeof value === "string") return value.trim().length > 0; if (!value || typeof value !== "object") return false; @@ -26,9 +30,12 @@ function hasContent(value: unknown): boolean { "tool_calls", "functionCall", "function_call", + "function", "arguments", - "input", "partial_json", + "id", + "name", + "input", "parts", ].some((key) => hasContent(object[key])); } @@ -141,10 +148,21 @@ export class DiscoveryValidityParser { private _ready = false; private _terminal = false; private _error = false; + private _limitExceeded = false; + private bytesSeen = 0; + private eventsSeen = 0; constructor(readonly protocol: DiscoveryProtocol) {} push(chunk: Uint8Array | string): DiscoveryValidity { + this.bytesSeen += + typeof chunk === "string" ? new TextEncoder().encode(chunk).byteLength : chunk.byteLength; + if (!this._ready && this.bytesSeen > DISCOVERY_PREFIX_MAX_BYTES) { + this._error = true; + this._limitExceeded = true; + this.buffered = ""; + return this.result; + } this.buffered += typeof chunk === "string" ? chunk : this.decoder.decode(chunk, { stream: true }); @@ -177,7 +195,7 @@ export class DiscoveryValidityParser { } } - return { ready: this._ready && !this._error, terminal: this._terminal, error: this._error }; + return this.result; } private consumeLine(line: string): void { @@ -195,6 +213,12 @@ export class DiscoveryValidityParser { } private consumeValue(value: unknown): void { + this.eventsSeen += 1; + if (!this._ready && this.eventsSeen > DISCOVERY_EVENT_MAX_COUNT) { + this._error = true; + this._limitExceeded = true; + return; + } const result = classifyJson(value, this.protocol); this._ready ||= result.ready; this._terminal ||= result.terminal; @@ -210,4 +234,17 @@ export class DiscoveryValidityParser { get error(): boolean { return this._error; } + + get limitExceeded(): boolean { + return this._limitExceeded; + } + + private get result(): DiscoveryValidity { + return { + ready: this._ready && !this._error, + terminal: this._terminal, + error: this._error, + ...(this._limitExceeded ? { limitExceeded: true } : {}), + }; + } } diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 120b15dd9..e31a81b17 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -24,12 +24,17 @@ import { PROTECTED_AUTH_HEADER_NAMES } from "@/lib/custom-headers"; import { recordEndpointFailure, recordEndpointSuccess } from "@/lib/endpoint-circuit-breaker"; import { applyGeminiGoogleSearchOverrideWithAudit } from "@/lib/gemini/provider-overrides"; import { logger } from "@/lib/logger"; +import { + DiscoveryRequestMetrics, + recordDiscoveryControlEvent, +} from "@/lib/observability/discovery-metrics"; import { getEndpointFilterStats, getPreferredProviderEndpoints, } from "@/lib/provider-endpoints/endpoint-selector"; import { getGlobalAgentPool, getProxyAgentForProvider } from "@/lib/proxy-agent"; import { RateLimitService } from "@/lib/rate-limit/service"; +import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import { SessionManager } from "@/lib/session-manager"; import { detectUpstreamErrorFromSseOrJsonText, @@ -55,6 +60,7 @@ import { HeaderProcessor, resolveAnthropicAuthHeaders } from "../headers"; import { evaluateResponsesWsEligibility, getResponsesWsSessionId, + isWebsocketClientRequest, } from "../responses-ws/eligibility"; import { RESERVED_INTERNAL_HEADERS } from "../responses-ws/internal-secret"; import { markResponsesWsUnsupported } from "../responses-ws/unsupported-cache"; @@ -190,6 +196,21 @@ function applyProviderCustomHeaders( const RETRY_LIMITS = PROVIDER_LIMITS.MAX_RETRY_ATTEMPTS; const MAX_PROVIDER_SWITCHES = 20; // 保险栓：最多切换 20 次供应商（防止无限循环） +const DISCOVERY_LEASE_HANDOFF_GRACE_SECONDS = 5; + +function isDiscoveryRolloutEligible(keyId: number, sessionId: string, percent: number): boolean { + const normalizedPercent = Math.max(0, Math.min(100, Math.floor(percent))); + if (normalizedPercent === 0) return false; + if (normalizedPercent === 100) return true; + + // FNV-1a provides a deterministic bucket without persisting rollout state. + let hash = 0x811c9dc5; + for (const character of `${keyId}:${sessionId}`) { + hash ^= character.charCodeAt(0); + hash = Math.imul(hash, 0x01000193); + } + return (hash >>> 0) % 100 < normalizedPercent; +} type CacheTtlOption = CacheTtlPreference | null | undefined; @@ -199,6 +220,41 @@ type ProxySessionWithAttemptRuntime = ProxySession & { releaseAgent?: () => void; }; +type DiscoveryCancellationKind = + | "discovery_sla_timeout" + | "discovery_loser" + | "request_deadline" + | "client_abort"; + +class DiscoveryCancellationError extends Error { + readonly kind: DiscoveryCancellationKind; + + constructor(kind: DiscoveryCancellationKind) { + super(kind); + this.name = "DiscoveryCancellationError"; + this.kind = kind; + } +} + +class DiscoveryValidityLimitError extends Error { + constructor() { + super("Discovery response prefix exceeded the validation limit"); + this.name = "DiscoveryValidityLimitError"; + } +} + +type PreparedStreamingDiscovery = { + settings: SystemSettings; + bindingSnapshot: SessionBindingSnapshot; + requestStartedAt: number; + lease: { + sessionId: string; + keyId: number; + ownerToken: string; + ttlSeconds: number; + }; +}; + type StreamingHedgeAttempt = { provider: Provider; session: ProxySession; @@ -1199,8 +1255,18 @@ export class ProxyForwarder { throw new Error("代理上下文缺少供应商或鉴权信息"); } - if (await ProxyForwarder.shouldUseStreamingDiscovery(session)) { - const discoveryPromise = ProxyForwarder.sendStreamingWithDiscovery(session); + const requestStartedAt = Date.now(); + const discoverySettings = await getCachedSystemSettings(); + const preparedDiscovery = await ProxyForwarder.prepareStreamingDiscovery( + session, + discoverySettings, + requestStartedAt + ); + if (preparedDiscovery) { + const discoveryPromise = ProxyForwarder.sendStreamingWithDiscovery( + session, + preparedDiscovery + ); void discoveryPromise.catch(() => undefined); return await discoveryPromise; } @@ -1480,6 +1546,7 @@ export class ProxyForwarder { endpointId: activeEndpoint.endpointId, endpointUrl: endpointAudit.endpointUrl, upstreamStatusCode: response.status, + bindingIntent: session.isSessionBindingAllowed() ? undefined : "none", }); logger.info("ProxyForwarder: Streaming response received, deferring finalization", { @@ -1667,7 +1734,7 @@ export class ProxyForwarder { } // ⭐ 成功后绑定 session 到供应商（智能绑定策略） - if (session.sessionId) { + if (session.sessionId && session.isSessionBindingAllowed()) { // 使用智能绑定策略（故障转移优先 + 稳定性优化） const result = await SessionManager.updateSessionBindingSmart( session.sessionId, @@ -3257,6 +3324,15 @@ export class ProxyForwarder { syscall?: string; // 系统调用：如 'getaddrinfo'、'connect'、'read'、'write' }; + const externalAbortReason = externalAbortSignal?.reason; + if ( + externalAbortSignal?.aborted && + externalAbortReason instanceof DiscoveryCancellationError + ) { + cleanupCombinedSignal(); + throw externalAbortReason; + } + // ⭐ SSL 证书错误检测：标记 Agent 为不健康，下次请求将创建新 Agent const sslErrorCacheKey = proxyConfig?.cacheKey ?? directConnectionCacheKey; const sslErrorDispatcherId = proxyConfig?.dispatcherId ?? directConnectionDispatcherId; @@ -3811,11 +3887,7 @@ export class ProxyForwarder { private static shouldUseStreamingHedge(session: ProxySession): boolean { const endpointPolicy = ProxyForwarder.getEndpointPolicy(session); - const routing = session as ProxySession & { - routingMode?: string; - disableStreamingHedge?: boolean; - }; - if (routing.routingMode === "lease_conflict_single" || routing.disableStreamingHedge === true) { + if (session.isStreamingHedgeDisabled()) { return false; } return ( @@ -3826,34 +3898,99 @@ export class ProxyForwarder { ); } - private static async shouldUseStreamingDiscovery(session: ProxySession): Promise { - const settings = await getCachedSystemSettings(); + private static async prepareStreamingDiscovery( + session: ProxySession, + settings: SystemSettings, + requestStartedAt: number + ): Promise { if (settings.discoveryEnabled !== true) { - return false; - } - if ( - typeof SessionManager.getVersionedBindingCapabilityState !== "function" || - SessionManager.getVersionedBindingCapabilityState() !== "available" - ) { - return false; + return null; } const endpointPolicy = ProxyForwarder.getEndpointPolicy(session); const protocol = ProxyForwarder.discoveryProtocol(session); const message = session.request.message as Record; - const routing = session as ProxySession & { - routingMode?: string; - disableStreamingHedge?: boolean; - }; - return ( - endpointPolicy.allowRetry && - endpointPolicy.allowProviderSwitch && - message?.stream === true && - !endpointPolicy.bypassForwarderPreprocessing && - protocol !== "unknown" && - routing.routingMode !== "lease_conflict_single" && - routing.disableStreamingHedge !== true && - !session.isRawCrossProviderFallbackEnabled() + if ( + !endpointPolicy.allowRetry || + !endpointPolicy.allowProviderSwitch || + message?.stream !== true || + endpointPolicy.bypassForwarderPreprocessing || + protocol === "unknown" || + isWebsocketClientRequest(session.headers) || + session.isStreamingHedgeDisabled() || + session.isRawCrossProviderFallbackEnabled() + ) { + return null; + } + + const sessionId = session.sessionId; + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + if (!sessionId || keyId == null) { + return null; + } + if (!isDiscoveryRolloutEligible(keyId, sessionId, getEnvConfig().DISCOVERY_ROLLOUT_PERCENT)) { + return null; + } + + const capabilityState = await SessionManager.ensureVersionedBindingCapability(); + if (capabilityState !== "available") { + return null; + } + + let bindingSnapshot = session.getSessionBindingSnapshot(); + if ( + !bindingSnapshot || + bindingSnapshot.sessionId !== sessionId || + bindingSnapshot.keyId !== keyId + ) { + const binding = await SessionManager.getSessionBindingSnapshot(sessionId, keyId); + if (binding.status !== "ok") { + // A foreign or irreconcilable mirror must never be mutated by this + // request. Infrastructure unavailability still falls back to the + // established legacy wrapper. + if (binding.status === "conflict") session.setSessionBindingAllowed(false); + return null; + } + bindingSnapshot = binding.snapshot; + session.setSessionBindingSnapshot(binding.snapshot); + } + + const ttlSeconds = Math.max( + 1, + Math.ceil(Math.max(1, settings.racingTotalTimeoutMs ?? 60_000) / 1000) + ); + const lease = await SessionManager.acquireSessionDiscoveryLease( + sessionId, + keyId, + ttlSeconds + DISCOVERY_LEASE_HANDOFF_GRACE_SECONDS ); + if (lease.status !== "acquired") { + if (lease.status === "conflict") { + session.disableStreamingHedge(); + session.setSessionBindingAllowed(false); + logger.info("[Discovery] Lease conflict; routing request in single-upstream mode", { + sessionId, + keyId, + }); + recordDiscoveryControlEvent("lease_conflict", { + requestId: session.messageContext?.id ?? null, + sessionId, + keyId, + }); + } + return null; + } + + return { + settings, + bindingSnapshot, + requestStartedAt, + lease: { + sessionId, + keyId, + ownerToken: lease.ownerToken, + ttlSeconds, + }, + }; } private static discoveryProtocol(session: ProxySession): DiscoveryProtocol { @@ -4623,7 +4760,7 @@ export class ProxyForwarder { // A non-hedged request is finalized through response-handler. Updating // here as well would perform a duplicate binding read/CAS before the // stream has passed its final validation. - if (session.sessionId && isActualHedgeWin) { + if (session.sessionId && isActualHedgeWin && session.isSessionBindingAllowed()) { void (async () => { const bindingResult = await SessionManager.updateSessionBindingSmart( session.sessionId!, @@ -4672,6 +4809,7 @@ export class ProxyForwarder { upstreamStatusCode: attempt.response.status, isHedgeWinner: isActualHedgeWin, billHedgeLosers, + bindingIntent: session.isSessionBindingAllowed() ? undefined : "none", }); const response = new Response( @@ -4716,7 +4854,9 @@ export class ProxyForwarder { } if (checkResult.referenced) { - session.recordProviderSessionRef(provider.id); + session.recordProviderSessionRef(provider.id, { + retainOnSuccess: checkResult.tracked, + }); } } @@ -4837,32 +4977,36 @@ export class ProxyForwarder { * existing loser billing and retry semantics remain unchanged while the * feature is rolled out behind discoveryEnabled. */ - private static async sendStreamingWithDiscovery(session: ProxySession): Promise { + private static async sendStreamingWithDiscovery( + session: ProxySession, + prepared: PreparedStreamingDiscovery + ): Promise { const initialProvider = session.provider; if (!initialProvider) throw new Error("代理上下文缺少供应商"); - const settings = await getCachedSystemSettings(); - const concurrency = Math.max(1, Math.floor(settings.discoveryConcurrency ?? 2)); + const { settings, lease, requestStartedAt } = prepared; + const concurrency = Math.max(2, Math.floor(settings.discoveryConcurrency ?? 2)); const maxRounds = Math.max(1, Math.floor(settings.maxDiscoveryRounds ?? 2)); const discoverySlaMs = Math.max(1, settings.discoverySlaMs ?? 10_000); - const stickySlaMs = Math.max(discoverySlaMs, settings.stickySlaMs ?? 20_000); - const totalTimeoutMs = Math.max(stickySlaMs, settings.racingTotalTimeoutMs ?? 60_000); + // Respect the configured Sticky budget. The settings validator already + // checks the total pre-winner window; a shorter Sticky SLA is a valid + // deliberate choice and must not be silently expanded at runtime. + const stickySlaMs = Math.max(1, settings.stickySlaMs ?? 20_000); + const totalTimeoutMs = Math.max(1, settings.racingTotalTimeoutMs ?? 60_000); + const racingDeadlineAt = requestStartedAt + totalTimeoutMs; const protocol = ProxyForwarder.discoveryProtocol(session); + const rawCrossProviderFallbackEnabled = session.isRawCrossProviderFallbackEnabled(); const coordinator = new DiscoveryCoordinator({ concurrency, maxRounds }); - const bindingKeyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; - // Provider selection normally populates this snapshot for a reused Sticky. - // For a cold start, initialize it once so finalization can use generation - // CAS without another read during winner commit or timeout cleanup. - let bindingSnapshot = session.getSessionBindingSnapshot(); - if (!bindingSnapshot && session.sessionId && bindingKeyId != null) { - const binding = await SessionManager.getSessionBindingSnapshot( - session.sessionId, - bindingKeyId - ); - if (binding.status === "ok") { - bindingSnapshot = binding.snapshot; - session.setSessionBindingSnapshot(binding.snapshot); - } - } + const discoveryMetrics = new DiscoveryRequestMetrics( + { + requestId: session.messageContext?.id ?? null, + sessionId: lease.sessionId, + keyId: lease.keyId, + }, + requestStartedAt + ); + let bindingSnapshot: SessionBindingSnapshot = prepared.bindingSnapshot; + let bindingWriteAllowed = true; + let leaseTransferred = false; const attempts = new Map< string, StreamingHedgeAttempt & { @@ -4875,6 +5019,11 @@ export class ProxyForwarder { ready: boolean; round: number; readerTransferred: boolean; + readerCancelled: boolean; + providerSessionRefOwned: boolean; + providerSessionRefRetainOnSuccess: boolean; + providerSessionRefReleased: boolean; + cancellationKind: DiscoveryCancellationKind | null; } >(); const launched = new Set(); @@ -4889,75 +5038,172 @@ export class ProxyForwarder { let totalTimer: NodeJS.Timeout | null = null; let roundTimer: NodeJS.Timeout | null = null; let stickyTimer: NodeJS.Timeout | null = null; + let roundLaunchesInProgress = 0; + const roundLaunchIdleWaiters = new Set<() => void>(); + let fallbackPromotionBlocked = false; + const hasSticky = + session.shouldReuseProvider() && + !!session.sessionId && + bindingSnapshot.providerId === initialProvider.id; + let stickyProbeActive = hasSticky; + if (hasSticky) coordinator.startStickyProbe(); + const waitForRoundLaunches = (): Promise => { + if (roundLaunchesInProgress === 0) return Promise.resolve(); + return new Promise((resolve) => roundLaunchIdleWaiters.add(resolve)); + }; + const notifyRoundLaunchIdle = () => { + if (roundLaunchesInProgress !== 0) return; + for (const resolve of roundLaunchIdleWaiters) resolve(); + roundLaunchIdleWaiters.clear(); + }; const clearRoundTimer = () => { if (roundTimer) { clearTimeout(roundTimer); roundTimer = null; } }; - let executeCoordinatorAction: (action: DiscoveryAction) => Promise = async () => {}; + let executeCoordinatorAction: ( + action: DiscoveryAction, + terminalCancellationKind?: DiscoveryCancellationKind + ) => Promise = async () => {}; let resolveResult: ((result: { response?: Response; error?: Error }) => void) | null = null; const resultPromise = new Promise<{ response?: Response; error?: Error }>((resolve) => { resolveResult = resolve; }); - const releaseProviderRef = (providerId: number) => { - ProxyForwarder.releaseProviderSessionRef(session, providerId); + const releaseProviderRef = (attempt: (typeof winner & { id: string }) | null) => { + if (!attempt?.providerSessionRefOwned || attempt.providerSessionRefReleased) return; + attempt.providerSessionRefReleased = true; + ProxyForwarder.releaseProviderSessionRef(session, attempt.provider.id); }; - const cancelAttempt = (attempt: (typeof winner & { id: string }) | null, reason: string) => { + const getAttemptModelRedirect = (attempt: (typeof winner & { id: string }) | null) => { + if (!attempt) return undefined; + if (attempt.modelRedirect !== undefined) return attempt.modelRedirect; + const redirect = attempt.session.getCurrentModelRedirect(attempt.provider.id); + if (redirect) attempt.modelRedirect = structuredClone(redirect); + return attempt.modelRedirect; + }; + + const cleanupAttempt = ( + attempt: (typeof winner & { id: string }) | null, + cancellationKind: DiscoveryCancellationKind | null + ) => { if (attempt?.readerTransferred) return; - if (!attempt?.pending) return; + if (!attempt) return; attempt.pending = false; - try { - attempt.controller.abort(new Error(reason)); - } catch { - /* abort is best effort */ + if (cancellationKind && !attempt.cancellationKind) { + attempt.cancellationKind = cancellationKind; } - try { - const cancelPromise = attempt.reader?.cancel(reason); - cancelPromise?.catch(() => undefined); - } catch (error) { - logger.debug("[Discovery] Reader cancel failed", { reason, error }); + if (!attempt.controller.signal.aborted) { + try { + attempt.controller.abort( + cancellationKind + ? new DiscoveryCancellationError(cancellationKind) + : new Error("discovery_attempt_failed") + ); + } catch { + /* abort is best effort */ + } } - try { - attempt.releaseAgent?.(); - } catch { - /* release is idempotent */ + if (attempt.reader && !attempt.readerCancelled) { + attempt.readerCancelled = true; + try { + const cancelPromise = attempt.reader.cancel( + cancellationKind ?? "discovery_attempt_failed" + ); + cancelPromise.catch((error) => + discoveryMetrics.cancelFailed(attempt.id, attempt.provider.id, error) + ); + } catch (error) { + discoveryMetrics.cancelFailed(attempt.id, attempt.provider.id, error); + logger.debug("[Discovery] Reader cancel failed", { cancellationKind, error }); + } + } + if (attempt.releaseAgent && !attempt.agentReleased) { + attempt.agentReleased = true; + try { + attempt.releaseAgent(); + } catch { + /* release is idempotent */ + } } - releaseProviderRef(attempt.provider.id); + releaseProviderRef(attempt); + attempt.chunks.length = 0; + discoveryMetrics.attemptFinished(attempt.id, { + providerId: attempt.provider.id, + outcome: cancellationKind ? "cancelled" : "failed", + cancellationKind, + }); }; - const cancelLosers = (keep: typeof winner = null) => { + const cancelAttempt = ( + attempt: (typeof winner & { id: string }) | null, + cancellationKind: DiscoveryCancellationKind + ) => cleanupAttempt(attempt, cancellationKind); + + const cancelLosers = ( + keep: typeof winner = null, + cancellationKind: DiscoveryCancellationKind = "discovery_loser" + ) => { for (const attempt of attempts.values()) { - if (attempt !== keep) cancelAttempt(attempt, "discovery_loser"); + if (attempt !== keep) cancelAttempt(attempt, cancellationKind); } }; - const settleFailure = async (error: Error) => { + const settleFailure = async ( + error: Error, + options: { + preserveBinding?: boolean; + cancellationKind?: DiscoveryCancellationKind; + } = {} + ) => { if (settled) return; settled = true; if (totalTimer) clearTimeout(totalTimer); if (roundTimer) clearTimeout(roundTimer); if (stickyTimer) clearTimeout(stickyTimer); - cancelLosers(); - if (bindingSnapshot) { + cancelLosers(null, options.cancellationKind ?? "discovery_loser"); + if (!options.preserveBinding && bindingWriteAllowed && session.isSessionBindingAllowed()) { if (bindingSnapshot.providerId != null) { - await SessionManager.clearVersionedSessionProvider( + void SessionManager.clearVersionedSessionProvider( bindingSnapshot, bindingSnapshot.providerId, 0 - ); + ).catch((bindingError) => { + logger.warn("[Discovery] Terminal binding clear failed", { + sessionId: bindingSnapshot.sessionId, + keyId: bindingSnapshot.keyId, + providerId: bindingSnapshot.providerId, + error: bindingError instanceof Error ? bindingError.message : String(bindingError), + }); + }); } - } else { - const attempted = new Set(launched); - await ProxyForwarder.clearSessionProviderBindings(session, attempted); } + const statusCode = error instanceof ProxyError ? error.statusCode : 503; + discoveryMetrics.finish({ + outcome: + options.cancellationKind === "client_abort" || statusCode === 499 + ? "client_abort" + : options.cancellationKind === "request_deadline" + ? "deadline" + : "failed", + statusCode, + winnerOrigin: "none", + }); resolveResult?.({ error }); }; const commit = async (attempt: typeof winner) => { - if (!attempt || committed || settled || !attempt.response || !attempt.reader) return; + if ( + !attempt || + committed || + settled || + !attempt.ready || + !attempt.response || + !attempt.reader + ) + return; committed = true; winner = attempt; attempt.pending = false; @@ -4968,6 +5214,20 @@ export class ProxyForwarder { if (roundTimer) clearTimeout(roundTimer); if (stickyTimer) clearTimeout(stickyTimer); cancelLosers(attempt); + discoveryMetrics.attemptFinished(attempt.id, { + providerId: attempt.provider.id, + outcome: "winner", + }); + discoveryMetrics.finish({ + outcome: "success", + statusCode: attempt.response.status, + winnerOrigin: attempt.kind, + winnerProviderId: attempt.provider.id, + winnerRound: + hasSticky && attempt.provider.id === initialProvider.id && stickyProbeActive + ? 0 + : attempt.round, + }); session.setProvider(attempt.provider); if (attempt.session !== session) ProxyForwarder.syncWinningAttemptSession(session, attempt.session); @@ -4986,14 +5246,18 @@ export class ProxyForwarder { isHedgeWinner: false, billHedgeLosers: false, bindingIntent: - attempt.kind === "fallback" + attempt.kind === "fallback" || !bindingWriteAllowed || !session.isSessionBindingAllowed() ? "none" : bindingSnapshot?.providerId == null ? "create" : "renew", bindingSnapshot, requiresCompletionMarker: attempt.kind !== "fallback", + discoveryLease: lease, + providerSessionRefOwned: attempt.providerSessionRefOwned, + providerSessionRefRetainOnSuccess: attempt.providerSessionRefRetainOnSuccess, }); + leaseTransferred = true; const prefix = attempt.chunks.length === 1 ? attempt.chunks[0] @@ -5032,11 +5296,94 @@ export class ProxyForwarder { return candidates[0]; }; - const launch = async (provider: Provider, kind: "normal" | "fallback"): Promise => { - if (settled || committed || launched.has(provider.id)) return; + const clearCapturedStickyBinding = async (cooldownTtlSeconds: number): Promise => { + if ( + !bindingWriteAllowed || + !session.isSessionBindingAllowed() || + bindingSnapshot.providerId !== initialProvider.id + ) { + return; + } + try { + const cleared = await SessionManager.clearVersionedSessionProvider( + bindingSnapshot, + initialProvider.id, + cooldownTtlSeconds + ); + if (cleared.status === "ok") { + bindingSnapshot = cleared.snapshot; + session.setSessionBindingSnapshot(cleared.snapshot); + return; + } + bindingWriteAllowed = false; + logger.debug("[Discovery] Failed to clear captured Sticky", { + providerId: initialProvider.id, + reason: cleared.reason, + }); + } catch (error) { + bindingWriteAllowed = false; + logger.debug("[Discovery] Failed to clear captured Sticky", { + providerId: initialProvider.id, + error, + }); + } + }; + + const scheduleRoundBoundary = (delayMs: number) => { + clearRoundTimer(); + const epoch = coordinator.epochs; + const remainingMs = Math.max(0, racingDeadlineAt - Date.now()); + roundTimer = setTimeout( + () => { + if (Date.now() >= racingDeadlineAt) { + void executeCoordinatorAction(coordinator.onDeadline(), "request_deadline").catch( + (error) => logger.warn("[Discovery] Deadline action failed", { error }) + ); + return; + } + void executeCoordinatorAction( + coordinator.onRoundBoundary(epoch.requestEpoch, epoch.roundEpoch) + ).catch((error) => logger.warn("[Discovery] Round boundary failed", { error })); + }, + Math.min(delayMs, remainingMs) + ); + }; + + const launch = async ( + provider: Provider, + kind: "normal" | "fallback", + options?: { + attemptSession?: ProxySession; + requestAttemptCount?: number; + retryState?: ReactiveRectifierRetryState; + providerSessionRefTransfer?: { + owned: boolean; + retainOnSuccess: boolean; + }; + } + ): Promise => { + const transferredProviderSessionRef = options?.providerSessionRefTransfer?.owned === true; + let providerSessionRefTracked = transferredProviderSessionRef; + let providerSessionRefRetainOnSuccess = + options?.providerSessionRefTransfer?.retainOnSuccess === true; + const rollbackLaunch = () => { + if (providerSessionRefTracked) { + ProxyForwarder.releaseProviderSessionRef(session, provider.id); + providerSessionRefTracked = false; + providerSessionRefRetainOnSuccess = false; + } + }; + if (settled || committed || launched.has(provider.id)) { + rollbackLaunch(); + return; + } launched.add(provider.id); - let providerSessionRefRecorded = false; - if (provider.id !== initialProvider.id && session.sessionId) { + if (!transferredProviderSessionRef && provider.id === initialProvider.id) { + providerSessionRefTracked = session.hasProviderSessionRef(provider.id); + providerSessionRefRetainOnSuccess = + providerSessionRefTracked && session.shouldRetainProviderSessionRefOnSuccess(provider.id); + } + if (!providerSessionRefTracked && session.sessionId) { const limit = provider.limitConcurrentSessions || 0; const check = await RateLimitService.checkAndTrackProviderSession( provider.id, @@ -5044,27 +5391,47 @@ export class ProxyForwarder { limit ); if (!check.allowed) { - launched.delete(provider.id); throw new ProxyError(check.reason || "Provider concurrent limit reached", 503); } if (check.referenced) { - session.recordProviderSessionRef(provider.id); - providerSessionRefRecorded = true; + session.recordProviderSessionRef(provider.id, { + retainOnSuccess: check.tracked, + }); + providerSessionRefTracked = true; + providerSessionRefRetainOnSuccess = check.tracked; } } + if (settled || committed) { + rollbackLaunch(); + return; + } let endpoint: Awaited>; try { endpoint = await ProxyForwarder.resolveStreamingHedgeEndpoint(session, provider); } catch (error) { - launched.delete(provider.id); - if (providerSessionRefRecorded) releaseProviderRef(provider.id); + rollbackLaunch(); throw error; } - const attemptSession = - provider.id === initialProvider.id - ? session - : ProxyForwarder.createStreamingShadowSession(session, provider); - attemptSession.setProvider(provider); + if (settled || committed) { + rollbackLaunch(); + return; + } + let attemptSession: ProxySession; + try { + attemptSession = + options?.attemptSession ?? + (provider.id === initialProvider.id + ? session + : ProxyForwarder.createStreamingShadowSession(session, provider)); + attemptSession.setProvider(provider); + } catch (error) { + rollbackLaunch(); + throw error; + } + if (settled || committed) { + rollbackLaunch(); + return; + } const controller = new AbortController(); const id = `${provider.id}:${sequence + 1}`; const attempt = { @@ -5077,6 +5444,11 @@ export class ProxyForwarder { ready: false, round: currentRound, readerTransferred: false, + readerCancelled: false, + providerSessionRefOwned: providerSessionRefTracked, + providerSessionRefRetainOnSuccess, + providerSessionRefReleased: false, + cancellationKind: null, provider, session: attemptSession, baseUrl: endpoint.baseUrl, @@ -5086,8 +5458,8 @@ export class ProxyForwarder { clearResponseTimeout: null, firstByteTimeoutMs: 0, sequence: ++sequence, - requestAttemptCount: 1, - reactiveRectifierRetryState: { + requestAttemptCount: options?.requestAttemptCount ?? 1, + reactiveRectifierRetryState: options?.retryState ?? { thinkingSignatureRetried: false, thinkingBudgetRetried: false, thinkingEffortConflictRetried: false, @@ -5114,9 +5486,13 @@ export class ProxyForwarder { ready: boolean; round: number; readerTransferred: boolean; + readerCancelled: boolean; + providerSessionRefOwned: boolean; + providerSessionRefRetainOnSuccess: boolean; + providerSessionRefReleased: boolean; + cancellationKind: DiscoveryCancellationKind | null; }; - attempts.set(id, attempt); - coordinator.addAttempt({ + const registered = coordinator.addAttempt({ id, providerId: provider.id, priority: ProxyProviderResolver.resolveEffectivePriorityForSession(provider, session), @@ -5126,6 +5502,20 @@ export class ProxyForwarder { round: currentRound, launchOrder: attempt.sequence, }); + if (!registered || settled || committed) { + rollbackLaunch(); + return; + } + attempts.set(id, attempt); + discoveryMetrics.attemptStarted({ + attemptId: id, + providerId: provider.id, + round: + stickyProbeActive && provider.id === initialProvider.id && kind === "normal" + ? 0 + : currentRound, + kind, + }); void ProxyForwarder.doForward( attempt.session, @@ -5143,11 +5533,17 @@ export class ProxyForwarder { attempt.releaseAgent = runtime.releaseAgent ?? null; attempt.clearResponseTimeout?.(); attempt.response = response; + if (!attempt.pending || committed || settled) { + if (response.body && !attempt.reader) attempt.reader = response.body.getReader(); + cleanupAttempt(attempt, attempt.cancellationKind); + return; + } if (!response.body) throw new EmptyResponseError(provider.id, provider.name, "empty_body"); attempt.reader = response.body.getReader(); while (!committed && !settled && attempt.pending) { const item = await attempt.reader.read(); + if (attempt.readerTransferred || committed || settled || !attempt.pending) return; if (item.done) { // A ready candidate may have reached EOF while waiting for a // higher-priority attempt. Its buffered prefix remains a valid @@ -5161,10 +5557,30 @@ export class ProxyForwarder { // A single read can contain both deliverable content and the // protocol terminator. Terminal is only invalid when no content // was observed; otherwise the buffered candidate is complete. + if (validity.limitExceeded) { + discoveryMetrics.event("parser_limit", { + attemptId: id, + providerId: provider.id, + round: attempt.round, + }); + throw new DiscoveryValidityLimitError(); + } if (validity.error || (validity.terminal && !validity.ready)) throw new ProxyError("Invalid upstream discovery response", 502); if (!validity.ready) continue; attempt.ready = true; + if ( + attempt.kind === "fallback" && + (fallbackPromotionBlocked || roundLaunchesInProgress > 0) + ) { + // The next wave has been reserved but its normal attempts may + // still be awaiting selection/endpoint setup. Keep the fallback + // ready-held until those slots are registered or exhausted. + return; + } + // Record readiness even when the priority gate holds this attempt. + // The coordinator can then promote the buffered stream if the + // higher-priority attempt fails or the round closes. const action = coordinator.markReady(id); if (action.type === "commit_normal" || action.type === "promote_fallback") await commit(attempt); @@ -5180,25 +5596,183 @@ export class ProxyForwarder { }) .catch(async (error) => { if (committed || settled || !attempt.pending) return; - attempt.pending = false; - const failureAction = coordinator.markFailed(id); lastError = error instanceof Error ? error : new Error(String(error)); lastErrorCategory = await categorizeErrorAsync(lastError); + const errorMessage = + lastError instanceof ProxyError + ? lastError.getDetailedErrorMessage() + : lastError.message; + + if (attempt.endpointAudit.endpointId != null) { + const isTimeoutError = lastError instanceof ProxyError && lastError.statusCode === 524; + if (isTimeoutError || lastErrorCategory === ErrorCategory.SYSTEM_ERROR) { + await recordEndpointFailure(attempt.endpointAudit.endpointId, lastError).catch( + () => undefined + ); + } + } + + if (lastErrorCategory === ErrorCategory.CLIENT_ABORT) { + attempt.pending = false; + coordinator.cancelRequest(); + session.addProviderToChain(provider, { + ...attempt.endpointAudit, + reason: "client_abort", + attemptNumber: attempt.sequence, + errorMessage: "Client aborted request", + }); + cleanupAttempt(attempt, "client_abort"); + await settleFailure( + lastError instanceof ProxyError + ? lastError + : new ProxyError("Request aborted by client", 499, undefined, true), + { preserveBinding: true, cancellationKind: "client_abort" } + ); + return; + } + + if (lastErrorCategory === ErrorCategory.LOCAL_OVERLOAD) { + const admission = findDbPoolAdmissionError(lastError); + const safeAdmissionMessage = admission?.message ?? "Database pool admission exceeded"; + session.addProviderToChain(provider, { + ...attempt.endpointAudit, + reason: "system_error", + attemptNumber: attempt.sequence, + errorMessage: safeAdmissionMessage, + errorDetails: { + system: { + errorType: "DbPoolAdmissionError", + errorName: "DbPoolAdmissionError", + errorMessage: safeAdmissionMessage, + errorCode: admission?.code, + }, + request: buildRequestDetails(session), + }, + }); + cleanupAttempt(attempt, null); + await settleFailure(lastError, { preserveBinding: true }); + return; + } + + // A failure can race the async selection/endpoint setup of the wave + // that is meant to replace it. Wait until those reserved slots have + // either registered or rolled back before the coordinator decides + // whether another round is needed. + await waitForRoundLaunches(); + if (committed || settled || !attempt.pending) return; + + // Preserve the existing provider-local rectifier contract before + // classifying a 400 as terminal. The rectifier mutates the shadow + // request session, so retry the same attempt session rather than + // creating a fresh unrectified shadow from the parent session. + const rectifier = await tryApplyReactiveRectifier({ + provider, + requestSession: attempt.session, + persistSession: session, + errorMessage, + attemptNumber: attempt.requestAttemptCount, + retryAttemptNumber: attempt.requestAttemptCount + 1, + retryState: attempt.reactiveRectifierRetryState, + }); + if (rectifier.matched && rectifier.applied) { + const providerSessionRefTransfer = { + owned: attempt.providerSessionRefOwned && !attempt.providerSessionRefReleased, + retainOnSuccess: attempt.providerSessionRefRetainOnSuccess, + }; + // The provider-local retry keeps the same concurrency slot. Move + // ownership to the replacement launch before cleaning the failed + // transport so there is no release/reacquire race window. + if (providerSessionRefTransfer.owned) attempt.providerSessionRefOwned = false; + attempt.pending = false; + coordinator.removeAttempt(id); + cleanupAttempt(attempt, null); + session.addProviderToChain(provider, { + ...buildRetryFailedChainEntry( + provider, + attempt.endpointAudit, + attempt.requestAttemptCount, + lastError, + errorMessage, + rectifier.requestDetailsBeforeRectify, + rawCrossProviderFallbackEnabled + ), + modelRedirect: getAttemptModelRedirect(attempt), + }); + launched.delete(provider.id); + try { + await launch(provider, attempt.kind, { + attemptSession: attempt.session, + requestAttemptCount: attempt.requestAttemptCount + 1, + retryState: attempt.reactiveRectifierRetryState, + providerSessionRefTransfer, + }); + } catch (retryLaunchError) { + lastError = + retryLaunchError instanceof Error + ? retryLaunchError + : new Error(String(retryLaunchError)); + lastErrorCategory = await categorizeErrorAsync(lastError); + if (stickyProbeActive && provider.id === initialProvider.id) { + stickyProbeActive = false; + if (stickyTimer) { + clearTimeout(stickyTimer); + stickyTimer = null; + } + await clearCapturedStickyBinding(0); + coordinator.startDiscoveryAfterSticky(); + await launchNextRound(concurrency, true); + return; + } + await settleFailure( + ProxyForwarder.resolveHedgeTerminalError(lastError, lastErrorCategory) + ); + } + return; + } + + const failedStickyProbe = + stickyProbeActive && attempt.kind === "normal" && provider.id === initialProvider.id; + attempt.pending = false; + const failureAction = failedStickyProbe + ? ({ type: "none" } as const) + : coordinator.markFailed(id); + if (failedStickyProbe) coordinator.removeAttempt(id); session.addProviderToChain(provider, { ...attempt.endpointAudit, reason: "retry_failed", attemptNumber: attempt.sequence, statusCode: lastError instanceof ProxyError ? lastError.statusCode : undefined, - errorMessage: lastError.message, + errorMessage, }); if ( + !(lastError instanceof DiscoveryValidityLimitError) && lastErrorCategory === ErrorCategory.PROVIDER_ERROR && !(lastError instanceof ProxyError && lastError.statusCode === 404) ) { await recordFailure(provider.id, lastError).catch(() => undefined); } - attempt.releaseAgent?.(); - releaseProviderRef(provider.id); + cleanupAttempt(attempt, null); + if (lastErrorCategory === ErrorCategory.NON_RETRYABLE_CLIENT_ERROR) { + // Client/input errors are independent of the selected provider. + // Stop Discovery immediately so the same invalid request is not + // fanned out or masked by a later generic fallback error. + await settleFailure( + ProxyForwarder.resolveHedgeTerminalError(lastError, lastErrorCategory), + { preserveBinding: true } + ); + return; + } + if (failedStickyProbe) { + stickyProbeActive = false; + if (stickyTimer) { + clearTimeout(stickyTimer); + stickyTimer = null; + } + await clearCapturedStickyBinding(0); + coordinator.startDiscoveryAfterSticky(); + await launchNextRound(concurrency, true); + return; + } const actionOwnsNextStep = failureAction.type === "commit_normal" || failureAction.type === "promote_fallback" || @@ -5209,7 +5783,18 @@ export class ProxyForwarder { } if (!actionOwnsNextStep && !committed && !settled) { const replacement = await chooseCandidate(); - if (replacement) await launch(replacement, "normal"); + if (replacement) { + try { + await launch(replacement, "normal"); + } catch (launchError) { + lastError = + launchError instanceof Error ? launchError : new Error(String(launchError)); + // A single launch failure does not prove that the remaining + // candidate pool is exhausted. The current round boundary can + // still advance or retry selection from the remaining pool. + noMoreCandidates = false; + } + } } if ( Array.from(attempts.values()).every((candidate) => !candidate.pending) && @@ -5228,31 +5813,60 @@ export class ProxyForwarder { }); }; - const launchNextRound = async () => { + const launchNextRound = async (slots: number, coordinatorAlreadyAdvanced = false) => { if (settled || committed) return; + roundLaunchesInProgress += 1; clearRoundTimer(); - currentRound += 1; - if (currentRound > maxRounds) return; - coordinator.beginRound(); - const candidate = await chooseCandidate(); - if (candidate) { - try { - await launch(candidate, "normal"); - } catch (error) { - lastError = error instanceof Error ? error : new Error(String(error)); + try { + if (coordinatorAlreadyAdvanced) { + currentRound = coordinator.round; + } else { + const nextRound = coordinator.beginRound(); + currentRound = nextRound.round; } - } - if (!committed && !settled) { - clearRoundTimer(); - roundTimer = setTimeout(() => { - void onBoundary().catch((error) => - logger.warn("[Discovery] Round boundary failed", { error }) + if (currentRound > maxRounds || slots <= 0) return; + const candidates = await ProxyProviderResolver.pickDiscoveryProviders( + session, + slots, + Array.from(launched) + ); + if (candidates.length === 0) { + noMoreCandidates = true; + } + for (const candidate of candidates) { + try { + await launch(candidate, "normal"); + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + // Launch setup failures do not establish pool exhaustion. + noMoreCandidates = false; + } + } + const hasPendingAttempt = Array.from(attempts.values()).some((attempt) => attempt.pending); + if (!hasPendingAttempt) { + await settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); + return; + } + if (!committed && !settled) { + scheduleRoundBoundary(discoverySlaMs); + } + } finally { + roundLaunchesInProgress = Math.max(0, roundLaunchesInProgress - 1); + if (roundLaunchesInProgress === 0) { + notifyRoundLaunchIdle(); + fallbackPromotionBlocked = false; + const readyFallback = Array.from(attempts.values()).find( + (attempt) => attempt.pending && attempt.ready && attempt.kind === "fallback" ); - }, discoverySlaMs); + if (readyFallback && !committed && !settled) { + const action = coordinator.markReady(readyFallback.id); + if (action.type === "promote_fallback") await commit(readyFallback); + } + } } }; - executeCoordinatorAction = async (action) => { + executeCoordinatorAction = async (action, terminalCancellationKind) => { if (settled || committed) return; if (action.type === "cancel" || action.type === "launch") { const cancelIds = @@ -5263,15 +5877,18 @@ export class ProxyForwarder { // the action. Restore the transport-facing state long enough for the // exactly-once cancellation/release path to run. if (attempt && !attempt.readerTransferred) attempt.pending = true; - if (attempt) cancelAttempt(attempt, "discovery_round_boundary"); + if (attempt) cancelAttempt(attempt, "discovery_sla_timeout"); } if (action.promoteAttemptId) { const fallback = attempts.get(action.promoteAttemptId); - if (fallback) fallback.kind = "fallback"; - if (action.type === "cancel" && currentRound < maxRounds) { - await launchNextRound(); + if (fallback) { + fallback.kind = "fallback"; + discoveryMetrics.fallbackPromoted(fallback.id, fallback.provider.id, fallback.round); } } + // A final-round fallback may still be waiting for a protocol-valid + // prefix. Keep it alive; markReady will commit it when it becomes safe. + if (action.type === "cancel") return; } if (action.type === "commit_normal" || action.type === "promote_fallback") { const attempt = attempts.get(action.attemptId); @@ -5279,113 +5896,105 @@ export class ProxyForwarder { return; } if (action.type === "launch") { - await launchNextRound(); + await launchNextRound(action.slots, true); return; } if (action.type === "terminal_failure") { - await settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); + await settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError), { + cancellationKind: terminalCancellationKind, + }); return; } - if (action.type === "none") { - const fallbackPending = Array.from(attempts.values()).some( - (attempt) => attempt.pending && attempt.kind === "fallback" - ); - if (fallbackPending && currentRound < maxRounds) { - await launchNextRound(); - } - } - }; - - const onBoundary = async () => { - if (settled || committed) return; - await executeCoordinatorAction(coordinator.onRoundBoundary()); }; const cleanupAbort = bindClientAbortListener(session.clientAbortSignal, () => { if (settled || committed) return; - void settleFailure(new ProxyError("Request aborted by client", 499, undefined, true)).catch( - (error) => logger.warn("[Discovery] Client abort cleanup failed", { error }) - ); + if (stickyTimer) clearTimeout(stickyTimer); + coordinator.cancelRequest(); + void settleFailure(new ProxyError("Request aborted by client", 499, undefined, true), { + preserveBinding: true, + cancellationKind: "client_abort", + }).catch((error) => logger.warn("[Discovery] Client abort cleanup failed", { error })); }); - totalTimer = setTimeout(() => { - if (settled || committed) return; - void executeCoordinatorAction(coordinator.onDeadline()).catch((error) => - logger.warn("[Discovery] Deadline action failed", { error }) - ); - }, totalTimeoutMs); + totalTimer = setTimeout( + () => { + if (settled || committed) return; + void executeCoordinatorAction(coordinator.onDeadline(), "request_deadline").catch((error) => + logger.warn("[Discovery] Deadline action failed", { error }) + ); + }, + Math.max(0, racingDeadlineAt - Date.now()) + ); - try { - const hasSticky = session.shouldReuseProvider() && !!session.sessionId; + const orchestrate = async () => { + let initialLaunchFailed = false; try { await launch(initialProvider, "normal"); } catch (error) { + initialLaunchFailed = true; + stickyProbeActive = false; lastError = error instanceof Error ? error : new Error(String(error)); - const replacement = await chooseCandidate(); - if (replacement) { - try { - await launch(replacement, "normal"); - } catch (replacementError) { - lastError = - replacementError instanceof Error - ? replacementError - : new Error(String(replacementError)); - await settleFailure(ProxyForwarder.resolveHedgeTerminalError(lastError, null)); - } - } else { - await settleFailure(ProxyForwarder.resolveHedgeTerminalError(lastError, null)); - } } - const initial = hasSticky ? concurrency - 1 : Math.max(0, concurrency - 1); + if (settled || committed) return; + if (hasSticky) { - stickyTimer = setTimeout(() => { - const sticky = Array.from(attempts.values()).find( - (attempt) => attempt.pending && attempt.provider.id === initialProvider.id - ); - if (sticky) { - if (!coordinator.demoteToFallback(sticky.id)) return; - sticky.kind = "fallback"; - if (bindingSnapshot && bindingSnapshot.providerId === initialProvider.id) { - void SessionManager.clearVersionedSessionProvider( - bindingSnapshot, - initialProvider.id, - Math.ceil((settings.stickyTimeoutCooldownMs ?? 300_000) / 1000) - ).catch((error) => - logger.debug("[Discovery] Failed to clear timed-out Sticky", { error }) - ); - } - if (currentRound < maxRounds) { - void launchNextRound().catch((error) => - logger.warn("[Discovery] Sticky round launch failed", { error }) - ); - } else { - void onBoundary().catch((error) => - logger.warn("[Discovery] Sticky boundary failed", { error }) + if (initialLaunchFailed) { + await clearCapturedStickyBinding(0); + coordinator.startDiscoveryAfterSticky(); + await launchNextRound(concurrency, true); + } else { + stickyTimer = setTimeout( + () => { + if (!stickyProbeActive || settled || committed) return; + if (Date.now() >= racingDeadlineAt) { + void executeCoordinatorAction(coordinator.onDeadline(), "request_deadline").catch( + (error) => logger.warn("[Discovery] Deadline action failed", { error }) + ); + return; + } + const sticky = Array.from(attempts.values()).find( + (attempt) => attempt.pending && attempt.provider.id === initialProvider.id ); - } - } - }, stickySlaMs); - } else { - const candidates = await ProxyProviderResolver.pickDiscoveryProviders( - session, - initial, - Array.from(launched) - ); - if (candidates.length === 0) noMoreCandidates = true; - for (const provider of candidates) { - try { - await launch(provider, "normal"); - } catch (error) { - lastError = error instanceof Error ? error : new Error(String(error)); - } - } - clearRoundTimer(); - roundTimer = setTimeout(() => { - void onBoundary().catch((error) => - logger.warn("[Discovery] Round boundary failed", { error }) + if (sticky) { + if (!coordinator.demoteToFallback(sticky.id)) return; + stickyProbeActive = false; + coordinator.startDiscoveryAfterSticky(); + sticky.kind = "fallback"; + discoveryMetrics.fallbackPromoted(sticky.id, sticky.provider.id, 0); + fallbackPromotionBlocked = true; + if (bindingSnapshot && bindingSnapshot.providerId === initialProvider.id) { + void clearCapturedStickyBinding( + Math.ceil((settings.stickyTimeoutCooldownMs ?? 300_000) / 1000) + ).finally(() => { + void launchNextRound(Math.max(0, concurrency - 1), true).catch((error) => + logger.warn("[Discovery] Sticky round launch failed", { error }) + ); + }); + return; + } + void launchNextRound(Math.max(0, concurrency - 1), true).catch((error) => + logger.warn("[Discovery] Sticky round launch failed", { error }) + ); + } + }, + Math.min(stickySlaMs, Math.max(0, racingDeadlineAt - Date.now())) ); - }, discoverySlaMs); + } + } else { + const activeNormals = Array.from(attempts.values()).filter( + (attempt) => attempt.pending && attempt.kind === "normal" + ).length; + await launchNextRound(Math.max(0, concurrency - activeNormals), true); } + }; + + void orchestrate().catch(async (error) => { + const normalized = error instanceof Error ? error : new Error(String(error)); + await settleFailure(ProxyForwarder.resolveHedgeTerminalError(normalized, null)); + }); + + try { const result = await resultPromise; if (result.error) throw result.error; return result.response as Response; @@ -5394,6 +6003,29 @@ export class ProxyForwarder { if (totalTimer) clearTimeout(totalTimer); clearRoundTimer(); if (stickyTimer) clearTimeout(stickyTimer); + if (!leaseTransferred) { + void SessionManager.releaseSessionDiscoveryLease( + lease.sessionId, + lease.keyId, + lease.ownerToken + ) + .then((released) => { + if (released.status !== "released") { + logger.debug("[Discovery] Lease release skipped", { + sessionId: lease.sessionId, + keyId: lease.keyId, + status: released.status, + }); + } + }) + .catch((releaseError) => { + logger.warn("[Discovery] Lease release failed", { + sessionId: lease.sessionId, + keyId: lease.keyId, + error: releaseError instanceof Error ? releaseError.message : String(releaseError), + }); + }); + } } } @@ -5627,7 +6259,7 @@ export class ProxyForwarder { session: ProxySession, expectedProviderId: number | null ): Promise { - if (!session.sessionId) return; + if (!session.sessionId || !session.isSessionBindingAllowed()) return; const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; await SessionManager.clearSessionProvider(session.sessionId, expectedProviderId, keyId); } @@ -5636,7 +6268,7 @@ export class ProxyForwarder { session: ProxySession, expectedProviderIds: Iterable ): Promise { - if (!session.sessionId) return; + if (!session.sessionId || !session.isSessionBindingAllowed()) return; const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; await SessionManager.clearSessionProviders(session.sessionId, expectedProviderIds, keyId); } @@ -5665,7 +6297,13 @@ export class ProxyForwarder { session as { consumeProviderSessionRef?: (id: number) => boolean } ).consumeProviderSessionRef; if (!providerSessionRefConsumer?.call(session, providerId)) return false; - void RateLimitService.releaseProviderSession(providerId, session.sessionId); + void RateLimitService.releaseProviderSession(providerId, session.sessionId).catch((error) => { + logger.warn("ProxyForwarder: Failed to release Provider session reference", { + providerId, + sessionId: session.sessionId, + error: error instanceof Error ? error.message : String(error), + }); + }); return true; } diff --git a/src/app/v1/_lib/proxy/provider-selector.ts b/src/app/v1/_lib/proxy/provider-selector.ts index d4ce6fdbe..e66e18272 100644 --- a/src/app/v1/_lib/proxy/provider-selector.ts +++ b/src/app/v1/_lib/proxy/provider-selector.ts @@ -296,7 +296,9 @@ export class ProxyProviderResolver { // === 成功 === if (checkResult.referenced) { - session.recordProviderSessionRef(session.provider.id); + session.recordProviderSessionRef(session.provider.id, { + retainOnSuccess: checkResult.tracked, + }); } logger.debug("ProviderSelector: Session tracked atomically", { diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index 70d0909b5..a9ca3a7de 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -9,10 +9,12 @@ import { getEnvConfig } from "@/lib/config/env.schema"; import { getCachedSystemSettings } from "@/lib/config/system-settings-cache"; import { emitProxyLangfuseTrace } from "@/lib/langfuse/emit-proxy-trace"; import { logger } from "@/lib/logger"; +import { recordDiscoveryControlEvent } from "@/lib/observability/discovery-metrics"; import { requestCloudPriceTableSync } from "@/lib/price-sync/cloud-price-updater"; import { ProxyStatusTracker } from "@/lib/proxy-status-tracker"; import { RateLimitService } from "@/lib/rate-limit"; import { deleteLiveChain } from "@/lib/redis/live-chain-store"; +import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import { SessionManager } from "@/lib/session-manager"; import { SessionTracker } from "@/lib/session-tracker"; import { CODEX_1M_CONTEXT_TOKEN_THRESHOLD } from "@/lib/special-attributes"; @@ -58,6 +60,7 @@ import { isClientAbortError, isTransportError } from "./errors"; import type { ProxySession } from "./session"; import { consumeDeferredStreamingFinalization, + type DeferredStreamingFinalization, peekDeferredStreamingFinalization, } from "./stream-finalization"; @@ -95,8 +98,157 @@ function resolveStreamTaskStaleTimeoutMs(): number { const STREAM_FINALIZATION_MAX_MS = 120_000; const STREAM_FAILURE_PERSISTENCE_MAX_MS = 5_000; +const DISCOVERY_LEASE_RELEASE_MAX_MS = 5_000; const NON_STREAM_TERMINAL_PERSISTENCE_ERROR = Symbol("non_stream_terminal_persistence_error"); +type DiscoveryLeaseLifecycle = { + active: boolean; + ensureOwned: () => Promise; + release: () => Promise; +}; + +function isSessionBindingMutationAllowed(session: ProxySession): boolean { + const checker = (session as ProxySession & { isSessionBindingAllowed?: () => boolean }) + .isSessionBindingAllowed; + return checker?.call(session) !== false; +} + +function startDiscoveryLeaseLifecycle(session: ProxySession): DiscoveryLeaseLifecycle { + const lease = peekDeferredStreamingFinalization(session)?.discoveryLease; + if (!lease) { + return { + active: false, + ensureOwned: async () => true, + release: async () => undefined, + }; + } + + let renewalTimer: ReturnType | null = null; + let renewalInFlight: Promise | null = null; + let releasePromise: Promise | null = null; + let ownershipState: "unknown" | "owned" | "lost" = "unknown"; + + const stopRenewal = () => { + if (renewalTimer) { + clearInterval(renewalTimer); + renewalTimer = null; + } + }; + + const renew = (): Promise => { + if (releasePromise || ownershipState === "lost") return Promise.resolve(false); + if (renewalInFlight) return renewalInFlight; + + const operation = (async () => { + const result = await SessionManager.renewSessionDiscoveryLease( + lease.sessionId, + lease.keyId, + lease.ownerToken, + lease.ttlSeconds + ); + if (result.status !== "renewed") { + ownershipState = "lost"; + stopRenewal(); + logger.warn("[ResponseHandler] Discovery lease renewal stopped", { + sessionId: lease.sessionId, + keyId: lease.keyId, + status: result.status, + reason: "reason" in result ? result.reason : undefined, + }); + return false; + } + ownershipState = "owned"; + return true; + })() + .catch((error) => { + ownershipState = "lost"; + stopRenewal(); + logger.warn("[ResponseHandler] Discovery lease renewal failed", { + sessionId: lease.sessionId, + keyId: lease.keyId, + error: error instanceof Error ? error.message : String(error), + }); + return false; + }) + .finally(() => { + if (renewalInFlight === operation) renewalInFlight = null; + }); + renewalInFlight = operation; + return operation; + }; + + // Ownership transfers from the Forwarder to the finalizer without delaying + // the downstream response. Renew immediately so an expired/lost token is + // observed before any terminal Session binding mutation is attempted. + const handoffRenewal = renew(); + const renewalIntervalMs = Math.max(250, Math.floor((lease.ttlSeconds * 1000) / 3)); + renewalTimer = setInterval(() => { + void renew(); + }, renewalIntervalMs); + renewalTimer.unref?.(); + + return { + active: true, + ensureOwned: async () => { + if (!(await handoffRenewal) || releasePromise || ownershipState !== "owned") return false; + // Revalidate with the owner token at the mutation boundary. This runs in + // post-terminal side effects, so it cannot add latency to downstream TTFB. + return renew(); + }, + release: () => { + if (releasePromise) return releasePromise; + stopRenewal(); + releasePromise = (async () => { + try { + const result = await raceWithTimeout( + SessionManager.releaseSessionDiscoveryLease( + lease.sessionId, + lease.keyId, + lease.ownerToken + ), + DISCOVERY_LEASE_RELEASE_MAX_MS, + "discovery_lease_release_timeout" + ); + if (result.status !== "released") { + logger.debug("[ResponseHandler] Discovery lease release skipped", { + sessionId: lease.sessionId, + keyId: lease.keyId, + status: result.status, + reason: "reason" in result ? result.reason : undefined, + }); + } + } catch (error) { + logger.warn("[ResponseHandler] Discovery lease release failed", { + sessionId: lease.sessionId, + keyId: lease.keyId, + error: error instanceof Error ? error.message : String(error), + }); + } + })(); + return releasePromise; + }, + }; +} + +async function releaseOwnedProviderSessionRef( + session: ProxySession, + meta: DeferredStreamingFinalization | null, + retainAsBaseline: boolean +): Promise { + if (retainAsBaseline || meta?.providerSessionRefOwned !== true || !session.sessionId) return; + if (!session.consumeProviderSessionRef(meta.providerId)) return; + + try { + await RateLimitService.releaseProviderSession(meta.providerId, session.sessionId); + } catch (error) { + logger.warn("[ResponseHandler] Failed to release Discovery Provider session reference", { + sessionId: session.sessionId, + providerId: meta.providerId, + error: error instanceof Error ? error.message : String(error), + }); + } +} + type MessageRequestTerminalDetails = Parameters[1]; type NonStreamTerminalPersistenceError = Error & { [NON_STREAM_TERMINAL_PERSISTENCE_ERROR]: true; @@ -974,8 +1126,31 @@ function hasPositiveBillableTokens(usage: UsageMetrics | null): boolean { return tokens > 0; } -const FINISH_REASON_MARKER = /"finish_reason"\s*:\s*"[a-z_]+"/; -const GEMINI_FINISH_REASON_MARKER = /"finishReason"\s*:\s*"[A-Z_]+"/; +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function hasOpenAIChatCompletionMarker(data: unknown): boolean { + if (!isRecord(data) || !Array.isArray(data.choices)) return false; + return data.choices.some( + (choice) => + isRecord(choice) && + typeof choice.finish_reason === "string" && + choice.finish_reason.trim().length > 0 + ); +} + +function hasGeminiCompletionMarker(data: unknown, format: ProxySession["originalFormat"]): boolean { + if (!isRecord(data)) return false; + const payload = format === "gemini-cli" && isRecord(data.response) ? data.response : data; + if (!Array.isArray(payload.candidates)) return false; + return payload.candidates.some( + (candidate) => + isRecord(candidate) && + typeof candidate.finishReason === "string" && + candidate.finishReason.trim().length > 0 + ); +} /** * 判断流式响应文本中是否存在“与格式匹配的终止完成标记”，用以区分 @@ -984,16 +1159,38 @@ const GEMINI_FINISH_REASON_MARKER = /"finishReason"\s*:\s*"[A-Z_]+"/; * 仅 usage>0 不足以证明完成：Anthropic 在首个 `message_start` 即带 usage、 * Gemini 在中间事件即带 usageMetadata，截断流同样会出现正向 token。 */ -function hasStreamCompletionMarker(text: string): boolean { - if ( - text.includes("response.completed") || // OpenAI Responses / Codex - text.includes("message_stop") || // Anthropic Messages - text.includes("[DONE]") // OpenAI Chat Completions - ) { - return true; +function hasStreamCompletionMarker(text: string, format: ProxySession["originalFormat"]): boolean { + const events = parseSSEData(text); + + switch (format) { + case "response": + return events.some( + (event) => + event.event === "response.completed" && + isRecord(event.data) && + event.data.type === "response.completed" && + isRecord(event.data.response) + ); + case "claude": + return events.some( + (event) => + event.event === "message_stop" && + isRecord(event.data) && + event.data.type === "message_stop" + ); + case "openai": + return events.some( + (event) => + event.event === "message" && + ((typeof event.data === "string" && event.data.trim() === "[DONE]") || + hasOpenAIChatCompletionMarker(event.data)) + ); + case "gemini": + case "gemini-cli": + return events.some( + (event) => event.event === "message" && hasGeminiCompletionMarker(event.data, format) + ); } - // OpenAI chat / Gemini：非空 finish reason 标记最终块。 - return FINISH_REASON_MARKER.test(text) || GEMINI_FINISH_REASON_MARKER.test(text); } export async function resolveBillableUsageMetricsForCost( @@ -1117,6 +1314,12 @@ type FinalizeDeferredStreamingResult = { }; /** Circuit and Session side effects, committed after durable terminal details. */ commitSideEffects?: () => Promise; + /** Attempt-scoped ref cleanup; idempotent via ProxySession ownership consumption. */ + finalizeAttemptResources?: () => Promise; + /** Whether terminal helpers may create auxiliary Sticky bindings (for example Codex cache keys). */ + allowAuxiliarySessionBinding: boolean; + /** Discovery auxiliary bindings must wait for, and depend on, the primary generation CAS. */ + confirmAuxiliarySessionBinding: () => Promise; }; /** @@ -1142,28 +1345,131 @@ function finalizeDeferredStreamingFinalizationIfNeeded( upstreamStatusCode: number, streamEndedNormally: boolean, clientAborted: boolean, + discoveryLeaseLifecycle: DiscoveryLeaseLifecycle, abortReason?: string ): FinalizeDeferredStreamingResult { const meta = consumeDeferredStreamingFinalization(session); const provider = session.provider; const providerIdForPersistence = meta?.providerId ?? provider?.id ?? null; const clearSessionBinding = async () => { - if (!session.sessionId) return; + if (!session.sessionId || !isSessionBindingMutationAllowed(session)) return; const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; - if (meta?.bindingIntent === "none") return; - if (meta?.bindingSnapshot && keyId != null) { - await SessionManager.clearVersionedSessionProvider( + if (meta?.bindingIntent === "none" || meta?.bindingIntent === "create") return; + if (meta?.bindingIntent === "renew") { + // A client disconnect is not evidence that the Sticky Provider failed. + // Discovery renewals may only clear the exact binding snapshot that was + // observed before the request. + if (clientAborted) return; + if ( + !meta.bindingSnapshot || + keyId == null || + meta.bindingSnapshot.keyId !== keyId || + meta.bindingSnapshot.sessionId !== session.sessionId || + meta.bindingSnapshot.providerId !== meta.providerId + ) { + logger.debug("[ResponseHandler] Discovery binding clear skipped", { + sessionId: session.sessionId, + keyId, + expectedProviderId: meta.providerId, + reason: "missing_or_mismatched_snapshot", + }); + return; + } + if (!(await discoveryLeaseLifecycle.ensureOwned())) { + logger.warn( + "[ResponseHandler] Discovery binding clear skipped after lease ownership loss", + { + sessionId: meta.bindingSnapshot.sessionId, + keyId: meta.bindingSnapshot.keyId, + expectedProviderId: meta.bindingSnapshot.providerId, + } + ); + return; + } + const cleared = await SessionManager.clearVersionedSessionProvider( meta.bindingSnapshot, - meta.bindingSnapshot.providerId, + meta.providerId, 0 ); + if (cleared.status !== "ok") { + logger.debug("[ResponseHandler] Discovery binding clear skipped", { + sessionId: meta.bindingSnapshot.sessionId, + keyId: meta.bindingSnapshot.keyId, + expectedProviderId: meta.bindingSnapshot.providerId, + reason: cleared.reason, + }); + } return; } + + // Legacy deferred finalization has no explicit binding intent and keeps its + // pre-Discovery behavior. await SessionManager.clearSessionProvider(session.sessionId, providerIdForPersistence, keyId); }; + let retainProviderSessionRef = false; + const finalizeProviderSessionRef = () => + releaseOwnedProviderSessionRef(session, meta, retainProviderSessionRef); + + const compareAndSetDiscoveryBinding = async ( + snapshot: SessionBindingSnapshot, + providerId: number, + keyId: number + ) => { + if ( + !snapshot || + snapshot.keyId !== keyId || + (session.sessionId !== null && snapshot.sessionId !== session.sessionId) + ) + return { updated: false, reason: "missing_snapshot", details: "missing_snapshot" }; + + if (!(await discoveryLeaseLifecycle.ensureOwned())) { + return { + updated: false, + reason: "discovery_lease_not_owned", + details: "lease_lost_or_unavailable", + }; + } + + const cas = await SessionManager.compareAndSetSessionProvider(snapshot, providerId); + if (cas.status === "conflict") { + recordDiscoveryControlEvent("binding_cas_conflict", { + requestId: session.messageContext?.id ?? null, + sessionId: snapshot.sessionId, + keyId, + providerId, + reason: cas.reason, + }); + } + + return { + updated: cas.status === "ok", + reason: cas.status === "ok" ? "discovery_generation_cas" : cas.reason, + details: cas.status, + }; + }; + const isHedgeWinner = meta?.isHedgeWinner === true; const billHedgeLosers = meta?.billHedgeLosers === true; + const allowAuxiliarySessionBinding = + isSessionBindingMutationAllowed(session) && + (meta?.bindingIntent === undefined || (meta.bindingIntent !== "none" && !clientAborted)); + const hasDiscoveryBindingIntent = + meta?.bindingIntent === "create" || meta?.bindingIntent === "renew"; + let resolvePrimaryDiscoveryBinding: ((updated: boolean) => void) | null = null; + const primaryDiscoveryBinding = hasDiscoveryBindingIntent + ? new Promise((resolve) => { + resolvePrimaryDiscoveryBinding = resolve; + }) + : Promise.resolve(allowAuxiliarySessionBinding); + let primaryDiscoveryBindingSettled = false; + const settlePrimaryDiscoveryBinding = (updated: boolean) => { + if (primaryDiscoveryBindingSettled) return; + primaryDiscoveryBindingSettled = true; + resolvePrimaryDiscoveryBinding?.(updated); + }; + const confirmAuxiliarySessionBinding = async () => + allowAuxiliarySessionBinding && (await primaryDiscoveryBinding); // 仅在“上游 HTTP=200 且流自然结束”时做“假 200”检测： // - 非 200：HTTP 已经表明失败（无需额外启发式） @@ -1177,7 +1483,7 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const completionMarkerMissing = meta?.requiresCompletionMarker === true && streamEndedNormally && - !hasStreamCompletionMarker(allContent); + !hasStreamCompletionMarker(allContent, session.originalFormat); let clientAbortGateUsage: FinalizeDeferredStreamingResult["clientAbortGateUsage"]; const clientAbortCompleteSuccess = (() => { if (!clientAborted || upstreamStatusCode < 200 || upstreamStatusCode >= 300) { @@ -1196,7 +1502,7 @@ function finalizeDeferredStreamingFinalizationIfNeeded( // completion marker is present, proving the upstream finished before the // client stopped reading. Otherwise keep the pre-PR safe default (499, // unbilled). - if (!hasStreamCompletionMarker(allContent)) { + if (!hasStreamCompletionMarker(allContent, session.originalFormat)) { return false; } @@ -1262,6 +1568,16 @@ function finalizeDeferredStreamingFinalizationIfNeeded( // - 只返回“内部状态码 + 错误原因”，由调用方写入统计； // - 不在这里更新熔断/绑定（meta 缺失意味着 Forwarder 没有启用延迟结算；provider 缺失意味着无法归因）。 if (!meta || !provider) { + const commitSideEffects = + shouldClearSessionBindingOnFailure || meta?.providerSessionRefOwned === true + ? async () => { + try { + if (shouldClearSessionBindingOnFailure) await clearSessionBinding(); + } finally { + await finalizeProviderSessionRef(); + } + } + : undefined; return { effectiveStatusCode, errorMessage, @@ -1269,7 +1585,10 @@ function finalizeDeferredStreamingFinalizationIfNeeded( isHedgeWinner, billHedgeLosers, clientAbortGateUsage, - commitSideEffects: shouldClearSessionBindingOnFailure ? clearSessionBinding : undefined, + commitSideEffects, + finalizeAttemptResources: finalizeProviderSessionRef, + allowAuxiliarySessionBinding, + confirmAuxiliarySessionBinding, }; } @@ -1313,22 +1632,26 @@ function finalizeDeferredStreamingFinalizationIfNeeded( }); const commitSideEffects = async () => { - await clearSessionBinding(); + try { + await clearSessionBinding(); - if (!clientAborted && session.getEndpointPolicy().allowCircuitBreakerAccounting) { - try { - const { recordFailure } = await import("@/lib/circuit-breaker"); - await recordFailure(meta.providerId, new Error(errorMessage ?? "STREAM_ABORTED")); - } catch (cbError) { - logger.warn("[ResponseHandler] Failed to record streaming failure in circuit breaker", { - providerId: meta.providerId, - sessionId: session.sessionId ?? null, - error: cbError, - }); - } + if (!clientAborted && session.getEndpointPolicy().allowCircuitBreakerAccounting) { + try { + const { recordFailure } = await import("@/lib/circuit-breaker"); + await recordFailure(meta.providerId, new Error(errorMessage ?? "STREAM_ABORTED")); + } catch (cbError) { + logger.warn("[ResponseHandler] Failed to record streaming failure in circuit breaker", { + providerId: meta.providerId, + sessionId: session.sessionId ?? null, + error: cbError, + }); + } - // Stream aborts are key-level errors. The endpoint delivered HTTP 200, - // so only the Provider circuit is updated here. + // Stream aborts are key-level errors. The endpoint delivered HTTP 200, + // so only the Provider circuit is updated here. + } + } finally { + await finalizeProviderSessionRef(); } }; @@ -1340,6 +1663,9 @@ function finalizeDeferredStreamingFinalizationIfNeeded( billHedgeLosers, clientAbortGateUsage, commitSideEffects, + finalizeAttemptResources: finalizeProviderSessionRef, + allowAuxiliarySessionBinding, + confirmAuxiliarySessionBinding, }; } @@ -1354,18 +1680,22 @@ function finalizeDeferredStreamingFinalizationIfNeeded( }); const commitSideEffects = async () => { - await clearSessionBinding(); - if (session.getEndpointPolicy().allowCircuitBreakerAccounting) { - try { - const { recordFailure } = await import("@/lib/circuit-breaker"); - await recordFailure(meta.providerId, new Error(errorMessage ?? "STREAM_ABORTED")); - } catch (cbError) { - logger.warn("[ResponseHandler] Failed to record missing stream completion marker", { - providerId: meta.providerId, - sessionId: session.sessionId ?? null, - error: cbError, - }); + try { + await clearSessionBinding(); + if (session.getEndpointPolicy().allowCircuitBreakerAccounting) { + try { + const { recordFailure } = await import("@/lib/circuit-breaker"); + await recordFailure(meta.providerId, new Error(errorMessage ?? "STREAM_ABORTED")); + } catch (cbError) { + logger.warn("[ResponseHandler] Failed to record missing stream completion marker", { + providerId: meta.providerId, + sessionId: session.sessionId ?? null, + error: cbError, + }); + } } + } finally { + await finalizeProviderSessionRef(); } }; @@ -1377,6 +1707,9 @@ function finalizeDeferredStreamingFinalizationIfNeeded( billHedgeLosers, clientAbortGateUsage, commitSideEffects, + finalizeAttemptResources: finalizeProviderSessionRef, + allowAuxiliarySessionBinding, + confirmAuxiliarySessionBinding, }; } @@ -1413,23 +1746,27 @@ function finalizeDeferredStreamingFinalizationIfNeeded( }); const commitSideEffects = async () => { - await clearSessionBinding(); + try { + await clearSessionBinding(); - // 404 is RESOURCE_NOT_FOUND and must not penalize the Provider circuit. - if ( - effectiveStatusCode !== 404 && - session.getEndpointPolicy().allowCircuitBreakerAccounting - ) { - try { - const { recordFailure } = await import("@/lib/circuit-breaker"); - await recordFailure(meta.providerId, new Error(detected.code)); - } catch (cbError) { - logger.warn("[ResponseHandler] Failed to record fake-200 error in circuit breaker", { - providerId: meta.providerId, - sessionId: session.sessionId ?? null, - error: cbError, - }); + // 404 is RESOURCE_NOT_FOUND and must not penalize the Provider circuit. + if ( + effectiveStatusCode !== 404 && + session.getEndpointPolicy().allowCircuitBreakerAccounting + ) { + try { + const { recordFailure } = await import("@/lib/circuit-breaker"); + await recordFailure(meta.providerId, new Error(detected.code)); + } catch (cbError) { + logger.warn("[ResponseHandler] Failed to record fake-200 error in circuit breaker", { + providerId: meta.providerId, + sessionId: session.sessionId ?? null, + error: cbError, + }); + } } + } finally { + await finalizeProviderSessionRef(); } }; @@ -1441,6 +1778,9 @@ function finalizeDeferredStreamingFinalizationIfNeeded( billHedgeLosers, clientAbortGateUsage, commitSideEffects, + finalizeAttemptResources: finalizeProviderSessionRef, + allowAuxiliarySessionBinding, + confirmAuxiliarySessionBinding, }; } @@ -1471,22 +1811,26 @@ function finalizeDeferredStreamingFinalizationIfNeeded( }); const commitSideEffects = async () => { - await clearSessionBinding(); + try { + await clearSessionBinding(); - if ( - effectiveStatusCode !== 404 && - session.getEndpointPolicy().allowCircuitBreakerAccounting - ) { - try { - const { recordFailure } = await import("@/lib/circuit-breaker"); - await recordFailure(meta.providerId, new Error(errorMessage)); - } catch (cbError) { - logger.warn("[ResponseHandler] Failed to record non-200 error in circuit breaker", { - providerId: meta.providerId, - sessionId: session.sessionId ?? null, - error: cbError, - }); + if ( + effectiveStatusCode !== 404 && + session.getEndpointPolicy().allowCircuitBreakerAccounting + ) { + try { + const { recordFailure } = await import("@/lib/circuit-breaker"); + await recordFailure(meta.providerId, new Error(errorMessage)); + } catch (cbError) { + logger.warn("[ResponseHandler] Failed to record non-200 error in circuit breaker", { + providerId: meta.providerId, + sessionId: session.sessionId ?? null, + error: cbError, + }); + } } + } finally { + await finalizeProviderSessionRef(); } }; @@ -1498,6 +1842,9 @@ function finalizeDeferredStreamingFinalizationIfNeeded( billHedgeLosers, clientAbortGateUsage, commitSideEffects, + finalizeAttemptResources: finalizeProviderSessionRef, + allowAuxiliarySessionBinding, + confirmAuxiliarySessionBinding, }; } @@ -1517,42 +1864,47 @@ function finalizeDeferredStreamingFinalizationIfNeeded( } const commitSideEffects = async () => { - if (meta.endpointId != null) { + let primaryDiscoveryBindingUpdated = false; + try { + if (meta.endpointId != null) { + try { + const { recordEndpointSuccess } = await import("@/lib/endpoint-circuit-breaker"); + await recordEndpointSuccess(meta.endpointId); + } catch (endpointError) { + logger.warn("[ResponseHandler] Failed to record endpoint success (stream finalized)", { + endpointId: meta.endpointId, + providerId: meta.providerId, + error: endpointError, + }); + } + } + try { - const { recordEndpointSuccess } = await import("@/lib/endpoint-circuit-breaker"); - await recordEndpointSuccess(meta.endpointId); - } catch (endpointError) { - logger.warn("[ResponseHandler] Failed to record endpoint success (stream finalized)", { - endpointId: meta.endpointId, + const { recordSuccess } = await import("@/lib/circuit-breaker"); + await recordSuccess(meta.providerId); + } catch (cbError) { + logger.warn("[ResponseHandler] Failed to record streaming success in circuit breaker", { providerId: meta.providerId, - error: endpointError, + error: cbError, }); } - } - try { - const { recordSuccess } = await import("@/lib/circuit-breaker"); - await recordSuccess(meta.providerId); - } catch (cbError) { - logger.warn("[ResponseHandler] Failed to record streaming success in circuit breaker", { - providerId: meta.providerId, - error: cbError, - }); - } - - // Hedge winner: commitWinner() already performed session binding and chain logging. - if (meta.bindingIntent !== "none" && !meta.isHedgeWinner && session.sessionId) { - const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; - const result = - meta.bindingSnapshot && keyId != null - ? await SessionManager.compareAndSetSessionProvider( - meta.bindingSnapshot, - meta.providerId - ).then((cas) => ({ - updated: cas.status === "ok", - reason: cas.status === "ok" ? "discovery_generation_cas" : cas.reason, - details: cas.status, - })) + // A client abort may still be billable when a completion marker was + // already buffered, but it must never create or renew Sticky state. + if ( + meta.bindingIntent !== "none" && + !meta.isHedgeWinner && + !clientAborted && + session.sessionId && + isSessionBindingMutationAllowed(session) + ) { + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + const isDiscoveryBinding = + meta.bindingIntent === "create" || meta.bindingIntent === "renew"; + const result = isDiscoveryBinding + ? meta.bindingSnapshot && keyId != null + ? await compareAndSetDiscoveryBinding(meta.bindingSnapshot, meta.providerId, keyId) + : { updated: false, reason: "missing_snapshot", details: "missing_snapshot" } : await SessionManager.updateSessionBindingSmart( session.sessionId, meta.providerId, @@ -1562,48 +1914,56 @@ function finalizeDeferredStreamingFinalizationIfNeeded( keyId ); - if (result.updated) { - logger.info("[ResponseHandler] Session binding updated (stream finalized)", { - sessionId: session.sessionId, - providerId: meta.providerId, - providerName: meta.providerName, - priority: meta.providerPriority, - reason: result.reason, - details: result.details, - attemptNumber: meta.attemptNumber, - totalProvidersAttempted: meta.totalProvidersAttempted, - }); - } else { - logger.debug("[ResponseHandler] Session binding not updated (stream finalized)", { - sessionId: session.sessionId, - providerId: meta.providerId, - providerName: meta.providerName, - priority: meta.providerPriority, - reason: result.reason, - details: result.details, - }); - } + primaryDiscoveryBindingUpdated = isDiscoveryBinding && result.updated; + retainProviderSessionRef = + primaryDiscoveryBindingUpdated && meta.providerSessionRefRetainOnSuccess === true; - if (session.shouldTrackSessionObservability()) { - void SessionManager.updateSessionProvider(session.sessionId, { - providerId: meta.providerId, - providerName: meta.providerName, - }).catch((err) => { - logger.error( - "[ResponseHandler] Failed to update session provider info (stream finalized)", - { error: err } - ); - }); + if (result.updated) { + logger.info("[ResponseHandler] Session binding updated (stream finalized)", { + sessionId: session.sessionId, + providerId: meta.providerId, + providerName: meta.providerName, + priority: meta.providerPriority, + reason: result.reason, + details: result.details, + attemptNumber: meta.attemptNumber, + totalProvidersAttempted: meta.totalProvidersAttempted, + }); + } else { + logger.debug("[ResponseHandler] Session binding not updated (stream finalized)", { + sessionId: session.sessionId, + providerId: meta.providerId, + providerName: meta.providerName, + priority: meta.providerPriority, + reason: result.reason, + details: result.details, + }); + } + + if (session.shouldTrackSessionObservability()) { + void SessionManager.updateSessionProvider(session.sessionId, { + providerId: meta.providerId, + providerName: meta.providerName, + }).catch((err) => { + logger.error( + "[ResponseHandler] Failed to update session provider info (stream finalized)", + { error: err } + ); + }); + } } - } - logger.info("[ResponseHandler] Streaming request finalized as success", { - providerId: meta.providerId, - providerName: meta.providerName, - attemptNumber: meta.attemptNumber, - totalProvidersAttempted: meta.totalProvidersAttempted, - statusCode: meta.upstreamStatusCode, - }); + logger.info("[ResponseHandler] Streaming request finalized as success", { + providerId: meta.providerId, + providerName: meta.providerName, + attemptNumber: meta.attemptNumber, + totalProvidersAttempted: meta.totalProvidersAttempted, + statusCode: meta.upstreamStatusCode, + }); + } finally { + settlePrimaryDiscoveryBinding(primaryDiscoveryBindingUpdated); + await finalizeProviderSessionRef(); + } }; return { @@ -1614,6 +1974,9 @@ function finalizeDeferredStreamingFinalizationIfNeeded( billHedgeLosers, clientAbortGateUsage, commitSideEffects, + finalizeAttemptResources: finalizeProviderSessionRef, + allowAuxiliarySessionBinding, + confirmAuxiliarySessionBinding, }; } @@ -1869,7 +2232,7 @@ export class ProxyResponseHandler { } const postTerminalSideEffects: Array<() => Promise> = []; - if (session.sessionId) { + if (session.sessionId && isSessionBindingMutationAllowed(session)) { const sessionId = session.sessionId; postTerminalSideEffects.push(async () => { const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; @@ -2044,7 +2407,9 @@ export class ProxyResponseHandler { const sessionId = session.sessionId; postTerminalSideEffects.push(async () => { const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; - await SessionManager.clearSessionProvider(sessionId, provider.id, keyId); + if (isSessionBindingMutationAllowed(session)) { + await SessionManager.clearSessionProvider(sessionId, provider.id, keyId); + } const sessionUsagePayload: SessionUsageUpdate = { status: @@ -2178,7 +2543,8 @@ export class ProxyResponseHandler { statusCode >= 200 && statusCode < 300 && session.sessionId && - provider.id + provider.id && + isSessionBindingMutationAllowed(session) ) { try { const responseData = JSON.parse(responseText) as Record; @@ -2594,10 +2960,16 @@ export class ProxyResponseHandler { private static async handleStream(session: ProxySession, response: Response): Promise { const messageContext = session.messageContext; const provider = session.provider; + const discoveryLeaseLifecycle = startDiscoveryLeaseLifecycle(session); if (!messageContext || !provider || !response.body) { discardBeforeResponseBodySnapshot(session); releaseSessionAgent(session); + const deferredMeta = peekDeferredStreamingFinalization(session); + void (async () => { + await releaseOwnedProviderSessionRef(session, deferredMeta, false); + await discoveryLeaseLifecycle.release(); + })(); return response; } @@ -2695,16 +3067,26 @@ export class ProxyResponseHandler { let transportReleased = false; let commitSideEffectsScheduled = false; let latestCommitSideEffects: (() => Promise) | undefined; + let latestFinalizeAttemptResources: (() => Promise) | undefined; const scheduleCommitSideEffects = (effect: (() => Promise) | undefined) => { - if (!effect || commitSideEffectsScheduled) return; + if ( + (!effect && !latestFinalizeAttemptResources && !discoveryLeaseLifecycle.active) || + commitSideEffectsScheduled + ) + return; commitSideEffectsScheduled = true; + const finalizeAttemptResources = latestFinalizeAttemptResources; return schedulePostTerminalSideEffects({ taskId, providerId: provider.id, sessionId: session.sessionId, commit: async (signal) => { - if (signal.aborted) return; - await effect(); + try { + if (!signal.aborted) await effect?.(); + } finally { + await finalizeAttemptResources?.(); + await discoveryLeaseLifecycle.release(); + } }, }); }; @@ -2900,9 +3282,11 @@ export class ProxyResponseHandler { statusCode, streamEndedNormally, clientAborted, + discoveryLeaseLifecycle, abortReason ); latestCommitSideEffects = finalized.commitSideEffects; + latestFinalizeAttemptResources = finalized.finalizeAttemptResources; const finalizedUsage = await finalizeRequestStats( session, allContent, @@ -2970,9 +3354,11 @@ export class ProxyResponseHandler { statusCode, false, clientAborted, + discoveryLeaseLifecycle, abortReason ); latestCommitSideEffects = finalized.commitSideEffects; + latestFinalizeAttemptResources = finalized.finalizeAttemptResources; await finalizeRequestStats( session, @@ -3013,6 +3399,12 @@ export class ProxyResponseHandler { } } finally { releaseTransportResources(); + if (!commitSideEffectsScheduled) { + void (async () => { + await latestFinalizeAttemptResources?.(); + await discoveryLeaseLifecycle.release(); + })(); + } } }; @@ -3297,15 +3689,30 @@ export class ProxyResponseHandler { let terminalDetailsPersisted = false; let streamCommitSideEffectsScheduled = false; let latestStreamCommitSideEffects: Array<() => Promise> = []; + let latestStreamFinalizeAttemptResources: (() => Promise) | undefined; const scheduleStreamCommitSideEffects = () => { - if (latestStreamCommitSideEffects.length === 0 || streamCommitSideEffectsScheduled) return; + if ( + (latestStreamCommitSideEffects.length === 0 && + !latestStreamFinalizeAttemptResources && + !discoveryLeaseLifecycle.active) || + streamCommitSideEffectsScheduled + ) + return; streamCommitSideEffectsScheduled = true; const committedEffects = [...latestStreamCommitSideEffects]; + const finalizeAttemptResources = latestStreamFinalizeAttemptResources; return schedulePostTerminalSideEffects({ taskId, providerId: provider.id, sessionId: session.sessionId, - commit: (signal) => runPostTerminalSideEffects(committedEffects, signal), + commit: async (signal) => { + try { + await runPostTerminalSideEffects(committedEffects, signal); + } finally { + await finalizeAttemptResources?.(); + await discoveryLeaseLifecycle.release(); + } + }, }); }; let streamFailurePersistencePromise: Promise | null = null; @@ -3356,11 +3763,13 @@ export class ProxyResponseHandler { statusCode, streamEndedNormally, clientAborted, + discoveryLeaseLifecycle, abortReason ); latestStreamCommitSideEffects = finalized.commitSideEffects ? [finalized.commitSideEffects] : []; + latestStreamFinalizeAttemptResources = finalized.finalizeAttemptResources; const effectiveStatusCode = finalized.effectiveStatusCode; const streamErrorMessage = finalized.errorMessage; const providerIdForPersistence = finalized.providerIdForPersistence; @@ -3457,7 +3866,8 @@ export class ProxyResponseHandler { effectiveStatusCode >= 200 && effectiveStatusCode < 300 && session.sessionId && - provider.id + provider.id && + finalized.allowAuxiliarySessionBinding ) { try { const sseEvents = parseSSEData(allContent); @@ -3645,6 +4055,7 @@ export class ProxyResponseHandler { if (codexCacheBinding) { const { sessionId, promptCacheKey, providerId, keyId } = codexCacheBinding; postTerminalSideEffects.push(async () => { + if (!(await finalized.confirmAuxiliarySessionBinding())) return; try { await SessionManager.updateSessionWithCodexCacheKey( sessionId, @@ -4067,6 +4478,12 @@ export class ProxyResponseHandler { clearClientAbortDrainTimer(); clearIdleTimer(); // 清除静默期计时器（防止泄漏） releaseSessionAgent(session); + if (!streamCommitSideEffectsScheduled) { + void (async () => { + await latestStreamFinalizeAttemptResources?.(); + await discoveryLeaseLifecycle.release(); + })(); + } } }; diff --git a/src/app/v1/_lib/proxy/session.ts b/src/app/v1/_lib/proxy/session.ts index 18ff8bcae..0ef7901c1 100644 --- a/src/app/v1/_lib/proxy/session.ts +++ b/src/app/v1/_lib/proxy/session.ts @@ -121,6 +121,11 @@ export class ProxySession { // Session ID（用于会话粘性和并发限流） sessionId: string | null; + // Discovery lease conflicts must stay on a single upstream and must not + // mutate a binding owned by the in-flight discovery request. + private streamingHedgeDisabled = false; + private sessionBindingAllowed = true; + // 客户端 IP（由 ProxyAuthenticator 按系统设置的 ip_extraction_config 解析后写入） clientIp: string | null = null; @@ -203,9 +208,10 @@ export class ProxySession { */ private providersSnapshot: Provider[] | null = null; - // 本请求已通过 Provider 并发检查获得的引用。 - // 失败切换 provider 时只能释放这里记录过的引用，避免 hedge/fallback 释放未 acquire 的 Redis 计数。 - private providerSessionRefs = new Set(); + // 本请求已通过 Provider 并发检查获得的引用。tracked=true 表示这次 + // acquire 同时创建了 Provider Session 基线；Sticky CAS 成功时只有 + // 该引用可以保留，已有基线上的普通 attempt 引用必须在终态释放。 + private providerSessionRefs = new Map>(); // Snapshot captured during provider selection. Discovery reuses this exact // generation for timeout cleanup/finalization instead of performing a @@ -363,25 +369,50 @@ export class ProxySession { return this.sessionBindingSnapshot; } - recordProviderSessionRef(providerId: number): void { + recordProviderSessionRef(providerId: number, options: { retainOnSuccess?: boolean } = {}): void { if (!this.providerSessionRefs) { - this.providerSessionRefs = new Set(); + this.providerSessionRefs = new Map>(); } if (Number.isInteger(providerId) && providerId > 0) { - this.providerSessionRefs.add(providerId); + const refs = this.providerSessionRefs.get(providerId) ?? []; + refs.push({ retainOnSuccess: options.retainOnSuccess === true }); + this.providerSessionRefs.set(providerId, refs); } } consumeProviderSessionRef(providerId: number): boolean { - if (!this.providerSessionRefs?.has(providerId)) { - return false; - } - - this.providerSessionRefs.delete(providerId); + const refs = this.providerSessionRefs?.get(providerId); + if (!refs || refs.length === 0) return false; + refs.shift(); + if (refs.length === 0) this.providerSessionRefs.delete(providerId); return true; } + hasProviderSessionRef(providerId: number): boolean { + return (this.providerSessionRefs?.get(providerId)?.length ?? 0) > 0; + } + + shouldRetainProviderSessionRefOnSuccess(providerId: number): boolean { + return this.providerSessionRefs?.get(providerId)?.[0]?.retainOnSuccess === true; + } + + disableStreamingHedge(): void { + this.streamingHedgeDisabled = true; + } + + isStreamingHedgeDisabled(): boolean { + return this.streamingHedgeDisabled === true; + } + + setSessionBindingAllowed(allowed: boolean): void { + this.sessionBindingAllowed = allowed; + } + + isSessionBindingAllowed(): boolean { + return this.sessionBindingAllowed !== false; + } + setCacheTtlResolved(ttl: CacheTtlResolved | null): void { this.cacheTtlResolved = ttl; } diff --git a/src/app/v1/_lib/proxy/stream-finalization.ts b/src/app/v1/_lib/proxy/stream-finalization.ts index 497207f1f..1ba8f7ada 100644 --- a/src/app/v1/_lib/proxy/stream-finalization.ts +++ b/src/app/v1/_lib/proxy/stream-finalization.ts @@ -1,6 +1,13 @@ import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import type { ProxySession } from "./session"; +export type DeferredStreamingDiscoveryLease = { + sessionId: string; + keyId: number; + ownerToken: string; + ttlSeconds: number; +}; + /** * 流式响应（SSE）在“收到响应头”时无法确定成功与否： * - 上游可能返回 HTTP 200，但 body 是错误 JSON（假 200） @@ -41,6 +48,12 @@ export type DeferredStreamingFinalization = { bindingSnapshot?: SessionBindingSnapshot | null; /** Discovery winners must satisfy the protocol completion marker before binding. */ requiresCompletionMarker?: boolean; + /** Lease already acquired by Forwarder and owned until terminal side effects finish. */ + discoveryLease?: DeferredStreamingDiscoveryLease; + /** Whether this attempt owns a Provider concurrent-session reference. */ + providerSessionRefOwned?: boolean; + /** CAS success converts this attempt ref into the binding baseline when true. */ + providerSessionRefRetainOnSuccess?: boolean; }; const deferredMeta = new WeakMap(); diff --git a/src/lib/config/env.schema.ts b/src/lib/config/env.schema.ts index 80d56a986..29c66acca 100644 --- a/src/lib/config/env.schema.ts +++ b/src/lib/config/env.schema.ts @@ -183,6 +183,10 @@ export const EnvSchema = z.object({ // 超时后主动断开该输家连接，仅用已收到的内容尝试计费（通常计不出 -> 跳过）。 HEDGE_LOSER_DRAIN_TIMEOUT_MS: z.coerce.number().int().min(1000).default(120_000), + // Operational canary for the Discovery scheduler. The database feature + // switch remains authoritative; this percentage only narrows eligibility. + DISCOVERY_ROLLOUT_PERCENT: z.coerce.number().int().min(0).max(100).default(100), + DASHBOARD_LOGS_POLL_INTERVAL_MS: z.coerce.number().int().min(250).max(60000).default(5000), // Langfuse Observability (optional, auto-enabled when keys are set) diff --git a/src/lib/observability/discovery-metrics.ts b/src/lib/observability/discovery-metrics.ts new file mode 100644 index 000000000..16cd93135 --- /dev/null +++ b/src/lib/observability/discovery-metrics.ts @@ -0,0 +1,126 @@ +import { logger } from "@/lib/logger"; + +export type DiscoveryLifecycleEvent = + | "request_started" + | "attempt_started" + | "attempt_finished" + | "fallback_promoted" + | "parser_limit" + | "cancel_failed" + | "lease_conflict" + | "binding_cas_conflict" + | "request_finished"; + +export type DiscoveryWinnerOrigin = "normal" | "fallback" | "none"; + +type DiscoveryMetricIdentity = { + requestId: number | string | null; + sessionId: string; + keyId: number; +}; + +export function recordDiscoveryControlEvent( + event: "lease_conflict" | "binding_cas_conflict", + context: DiscoveryMetricIdentity & Record +): void { + logger.info("[DiscoveryMetric] Control event", { event, ...context }); +} + +export class DiscoveryRequestMetrics { + private readonly attemptStartedAt = new Map(); + private readonly fallbackAttempts = new Set(); + private attempts = 0; + private active = 0; + private maxActive = 0; + private maxRound = 0; + private providerMs = 0; + private fallbackPromotions = 0; + private cancelFailures = 0; + private finished = false; + + constructor( + private readonly identity: DiscoveryMetricIdentity, + private readonly startedAt: number + ) { + this.event("request_started"); + } + + event(event: DiscoveryLifecycleEvent, context: Record = {}): void { + logger.debug("[DiscoveryMetric] Lifecycle event", { + event, + ...this.identity, + elapsedMs: Math.max(0, Date.now() - this.startedAt), + ...context, + }); + } + + attemptStarted(options: { + attemptId: string; + providerId: number; + round: number; + kind: DiscoveryWinnerOrigin; + }): void { + if (this.attemptStartedAt.has(options.attemptId)) return; + this.attemptStartedAt.set(options.attemptId, Date.now()); + this.attempts += 1; + this.active += 1; + this.maxActive = Math.max(this.maxActive, this.active); + this.maxRound = Math.max(this.maxRound, options.round); + this.event("attempt_started", options); + } + + attemptFinished( + attemptId: string, + context: { providerId: number; outcome: string; cancellationKind?: string | null } + ): void { + const startedAt = this.attemptStartedAt.get(attemptId); + if (startedAt == null) return; + this.attemptStartedAt.delete(attemptId); + this.active = Math.max(0, this.active - 1); + const durationMs = Math.max(0, Date.now() - startedAt); + this.providerMs += durationMs; + this.event("attempt_finished", { attemptId, durationMs, ...context }); + } + + fallbackPromoted(attemptId: string, providerId: number, round: number): void { + if (this.fallbackAttempts.has(attemptId)) return; + this.fallbackAttempts.add(attemptId); + this.fallbackPromotions += 1; + this.maxRound = Math.max(this.maxRound, round); + this.event("fallback_promoted", { attemptId, providerId, round }); + } + + cancelFailed(attemptId: string, providerId: number, error: unknown): void { + this.cancelFailures += 1; + this.event("cancel_failed", { + attemptId, + providerId, + error: error instanceof Error ? error.message : String(error), + }); + } + + finish(context: { + outcome: "success" | "failed" | "client_abort" | "deadline"; + statusCode: number; + winnerOrigin?: DiscoveryWinnerOrigin; + winnerProviderId?: number | null; + winnerRound?: number | null; + }): void { + if (this.finished) return; + this.finished = true; + const elapsedMs = Math.max(0, Date.now() - this.startedAt); + logger.info("[DiscoveryMetric] Request aggregate", { + event: "request_finished", + ...this.identity, + ...context, + elapsedMs, + ttfbMs: context.outcome === "success" ? elapsedMs : null, + attemptsPerRequest: this.attempts, + maxActiveAttempts: this.maxActive, + rounds: this.maxRound, + providerMs: this.providerMs, + fallbackPromotions: this.fallbackPromotions, + cancelFailures: this.cancelFailures, + }); + } +} diff --git a/tests/integration/proxy-hedge-lifecycle.test.ts b/tests/integration/proxy-hedge-lifecycle.test.ts index 390727090..0f2b6b3b0 100644 --- a/tests/integration/proxy-hedge-lifecycle.test.ts +++ b/tests/integration/proxy-hedge-lifecycle.test.ts @@ -16,10 +16,45 @@ const state = vi.hoisted(() => { return { addLoserCost: vi.fn(), billHedgeLosers: false, - durableTerminal: vi.fn(async () => {}), + discoveryEnabled: false, + acquireDiscoveryLease: vi.fn(async () => ({ + status: "acquired", + ownerToken: "integration-lease", + legacyFallbackAllowed: false, + })), + releaseDiscoveryLease: vi.fn(async () => ({ + status: "released", + legacyFallbackAllowed: false, + })), + renewDiscoveryLease: vi.fn(async () => ({ + status: "renewed", + legacyFallbackAllowed: false, + })), + compareAndSetBinding: vi.fn(async () => ({ + status: "ok", + source: "updated", + legacyFallbackAllowed: false, + snapshot: { + sessionId: "integration-discovery", + keyId: 22, + providerId: 2, + generation: "g2", + }, + })), + durableTerminal: vi.fn( + async ( + _id: number, + details: unknown, + options?: { onCommitted?: (details: unknown) => void | Promise } + ) => { + await options?.onCommitted?.(details); + return true; + } + ), http2Error: ((): Error | null => null)(), loserBilled: Promise.withResolvers(), pickAlternative: vi.fn(), + pickDiscovery: vi.fn(), providers: Array.from([]), recordFailure: vi.fn(async () => {}), settleLeaseBudgets: vi.fn(async () => {}), @@ -42,10 +77,17 @@ vi.mock("@/lib/config", async (importOriginal) => { ...actual, getCachedSystemSettings: async () => ({ billHedgeLosers: state.billHedgeLosers, + discoveryConcurrency: 2, + discoveryEnabled: state.discoveryEnabled, + discoverySlaMs: 100, enableBillingHeaderRectifier: false, enableClaudeMetadataUserIdInjection: false, enableThinkingBudgetRectifier: false, enableThinkingSignatureRectifier: false, + maxDiscoveryRounds: 1, + racingTotalTimeoutMs: 500, + stickySlaMs: 100, + stickyTimeoutCooldownMs: 300_000, }), isHttp2Enabled: async () => { if (state.http2Error) throw state.http2Error; @@ -57,8 +99,41 @@ vi.mock("@/lib/config/system-settings-cache", () => ({ getCachedSystemSettings: async () => ({ billNonSuccessfulRequests: false }), })); vi.mock("@/app/v1/_lib/proxy/provider-selector", () => ({ - ProxyProviderResolver: { pickRandomProviderWithExclusion: state.pickAlternative }, + ProxyProviderResolver: { + pickDiscoveryProviders: state.pickDiscovery, + pickRandomProviderWithExclusion: state.pickAlternative, + resolveEffectivePriorityForSession: (provider: Provider) => provider.priority ?? 0, + }, })); +vi.mock("@/lib/session-manager", async (importOriginal) => { + const actual = await importOriginal(); + class TestSessionManager extends actual.SessionManager { + static override async ensureVersionedBindingCapability() { + return "available" as const; + } + static override async getSessionBindingSnapshot(sessionId: string, keyId: number) { + return { + status: "ok" as const, + source: "existing" as const, + legacyFallbackAllowed: false as const, + snapshot: { sessionId, keyId, providerId: null, generation: "g1" }, + }; + } + static override async acquireSessionDiscoveryLease() { + return state.acquireDiscoveryLease(); + } + static override async renewSessionDiscoveryLease() { + return state.renewDiscoveryLease(); + } + static override async releaseSessionDiscoveryLease() { + return state.releaseDiscoveryLease(); + } + static override async compareAndSetSessionProvider() { + return state.compareAndSetBinding(); + } + } + return { ...actual, SessionManager: TestSessionManager }; +}); vi.mock("@/lib/provider-endpoints/endpoint-selector", () => ({ getEndpointFilterStats: vi.fn(async () => null), getPreferredProviderEndpoints: vi.fn(async () => []), @@ -154,7 +229,10 @@ vi.mock("@/lib/session-tracker", () => ({ vi.mock("@/lib/proxy-status-tracker", () => ({ ProxyStatusTracker: { getInstance: () => ({ endRequest: vi.fn() }) }, })); -vi.mock("@/lib/redis/live-chain-store", () => ({ deleteLiveChain: vi.fn(async () => {}) })); +vi.mock("@/lib/redis/live-chain-store", () => ({ + deleteLiveChain: vi.fn(async () => {}), + writeLiveChain: vi.fn(async () => {}), +})); const CREATED_AT = new Date(0); const USER = { @@ -386,6 +464,7 @@ beforeEach(async () => { await resetGlobalAgentPool(); vi.clearAllMocks(); state.billHedgeLosers = false; + state.discoveryEnabled = false; state.http2Error = null; state.loserBilled = Promise.withResolvers(); state.providers.length = 0; @@ -394,6 +473,10 @@ beforeEach(async () => { state.pickAlternative.mockImplementation(async (_session: unknown, excludedIds: number[]) => { return state.providers.find((provider) => !excludedIds.includes(provider.id)) ?? null; }); + state.pickDiscovery.mockImplementation( + async (_session: unknown, count: number, excludedIds: number[]) => + state.providers.filter((provider) => !excludedIds.includes(provider.id)).slice(0, count) + ); }); afterEach(async () => { @@ -403,6 +486,44 @@ afterEach(async () => { }); describe("proxy hedge transport/lifecycle integration (persistence and control-plane seams mocked)", () => { + it("runs a leased Discovery race over real loopback transports and cancels the loser", async () => { + const [loser, winner] = await Promise.all([startUpstream(), startUpstream()]); + const client = new AbortController(); + try { + state.discoveryEnabled = true; + const initialProvider = createProvider(1, loser.baseUrl, 0); + const winningProvider = createProvider(2, winner.baseUrl, 0); + winningProvider.priority = initialProvider.priority; + state.providers.push(winningProvider); + const session = await createSession(initialProvider, "/v1/messages", client.signal); + session.sessionId = "integration-discovery"; + const agents = watchAgentReleases(2); + + const forwarded = ProxyForwarder.send(session); + await Promise.all([loser.response, winner.response]); + await winner.send( + 'event: content_block_delta\ndata: {"type":"content_block_delta","delta":{"type":"text_delta","text":"winner"}}\n\n' + + 'event: message_stop\ndata: {"type":"message_stop"}\n\n' + ); + + const downstream = await ProxyResponseHandler.dispatch(session, await forwarded); + await expect(downstream.text()).resolves.toContain("winner"); + await settleTasks(); + await loser.terminated; + await agents.released; + + expect(loser.abortCount()).toBe(1); + expect(winner.abortCount()).toBe(0); + expect(state.acquireDiscoveryLease).toHaveBeenCalledTimes(1); + expect(state.compareAndSetBinding).toHaveBeenCalledTimes(1); + expect(state.releaseDiscoveryLease).toHaveBeenCalledTimes(1); + expect(agents.pool.getPoolStats().activeRequests).toBe(0); + } finally { + client.abort(new Error("fixture cleanup")); + await Promise.all([loser.close(), winner.close()]); + } + }); + it("fences loser timers after winner settlement and releases each launched transport once", async () => { const [slow, winner, fenced] = await Promise.all([ startUpstream(), diff --git a/tests/unit/proxy/discovery-coordinator.test.ts b/tests/unit/proxy/discovery-coordinator.test.ts index 3a41a8d5e..bdbb7ff20 100644 --- a/tests/unit/proxy/discovery-coordinator.test.ts +++ b/tests/unit/proxy/discovery-coordinator.test.ts @@ -13,6 +13,20 @@ const attempt = (id: string, priority: number, kind: "normal" | "fallback" = "no }); describe("DiscoveryCoordinator", () => { + it("keeps Sticky probing outside the Discovery round counter", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 1 }); + coordinator.startStickyProbe(); + expect(coordinator.state).toBe("STICKY_PROBING"); + expect(coordinator.round).toBe(1); + + coordinator.addAttempt(attempt("sticky", 1)); + expect(coordinator.demoteToFallback("sticky")).toBe(true); + coordinator.startDiscoveryAfterSticky(); + + expect(coordinator.state).toBe("DISCOVERY_RACING"); + expect(coordinator.round).toBe(1); + }); + it("commits the highest priority ready normal attempt", () => { const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); coordinator.addAttempt(attempt("a", 10)); @@ -44,6 +58,7 @@ describe("DiscoveryCoordinator", () => { expect(coordinator.markReady("a", epoch.requestEpoch, epoch.roundEpoch)).toEqual({ type: "none", }); + expect(coordinator.markFailed("a")).toEqual({ type: "none" }); }); it("promotes a ready fallback at the round boundary when no normal is ready", () => { @@ -100,6 +115,49 @@ describe("DiscoveryCoordinator", () => { ready: true, pending: true, }); - expect(coordinator.markFailed("high")).toEqual({ type: "commit_normal", attemptId: "low" }); + expect(coordinator.markFailed("high")).toEqual({ + type: "commit_normal", + attemptId: "low", + }); + }); + + it("treats fallback promotion as terminal", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("fallback", 1, "fallback")); + + expect(coordinator.markReady("fallback")).toEqual({ + type: "promote_fallback", + attemptId: "fallback", + }); + expect(coordinator.state).toBe("FALLBACK_ACTIVE"); + expect(coordinator.isTerminal).toBe(true); + expect(coordinator.markFailed("fallback")).toEqual({ type: "none" }); + }); + + it("keeps coordinator kind in sync when a running Sticky becomes fallback", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("sticky", 1)); + + expect(coordinator.promoteToFallback("sticky")).toBe(true); + expect(coordinator.snapshot.find((item) => item.id === "sticky")?.kind).toBe("fallback"); + }); + + it("opens a full new round when all normal attempts fail", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 3, maxRounds: 2 }); + coordinator.addAttempt(attempt("a", 1)); + coordinator.addAttempt(attempt("b", 2)); + + expect(coordinator.markFailed("a")).toEqual({ type: "none" }); + expect(coordinator.markFailed("b")).toEqual({ type: "launch", slots: 3 }); + expect(coordinator.round).toBe(2); + }); + + it("commits a ready normal candidate at the total deadline", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("high", 1)); + coordinator.addAttempt(attempt("normal", 2)); + coordinator.markReady("normal"); + + expect(coordinator.onDeadline()).toEqual({ type: "commit_normal", attemptId: "normal" }); }); }); diff --git a/tests/unit/proxy/discovery-validity.test.ts b/tests/unit/proxy/discovery-validity.test.ts index 7a6f95fc4..9c5ca8b39 100644 --- a/tests/unit/proxy/discovery-validity.test.ts +++ b/tests/unit/proxy/discovery-validity.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from "vitest"; import { + DISCOVERY_EVENT_MAX_COUNT, + DISCOVERY_PREFIX_MAX_BYTES, DiscoveryValidityParser, classifyDiscoveryChunk, } from "@/app/v1/_lib/proxy/discovery-validity"; @@ -87,6 +89,27 @@ describe("discovery validity", () => { }); }); + it("accepts Anthropic tool-use partial JSON as deliverable content", () => { + const parser = new DiscoveryValidityParser("anthropic"); + + expect( + parser.push('data: {"type":"content_block_delta","delta":{"partial_json":"{\\"x\\":1}"}}\n\n') + ).toMatchObject({ ready: true, error: false }); + expect(parser.push('data: {"type":"message_stop"}\n\n')).toMatchObject({ + ready: true, + terminal: true, + error: false, + }); + }); + + it("accepts nested OpenAI Chat tool-call arguments", () => { + expect( + parserForOpenAIChatToolCall().push( + 'data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"{\\"x\\":1}"}}]}}]}\n\n' + ) + ).toMatchObject({ ready: true, error: false }); + }); + it("accepts Anthropic tool-use starts and partial JSON deltas", () => { expect( classifyDiscoveryChunk( @@ -101,4 +124,23 @@ describe("discovery validity", () => { ).ready ).toBe(true); }); + + it("fails a metadata-only prefix after the byte limit", () => { + const parser = new DiscoveryValidityParser("openai-chat"); + const result = parser.push(`:${"x".repeat(DISCOVERY_PREFIX_MAX_BYTES + 1)}`); + expect(result).toMatchObject({ ready: false, error: true, limitExceeded: true }); + }); + + it("fails metadata-only protocol events after the event limit", () => { + const parser = new DiscoveryValidityParser("anthropic"); + let result = parser.push(""); + for (let index = 0; index <= DISCOVERY_EVENT_MAX_COUNT; index += 1) { + result = parser.push('data: {"type":"ping"}\n'); + } + expect(result).toMatchObject({ ready: false, error: true, limitExceeded: true }); + }); }); + +function parserForOpenAIChatToolCall(): DiscoveryValidityParser { + return new DiscoveryValidityParser("openai-chat"); +} diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index 5654e4745..bd2cafab4 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -42,6 +42,33 @@ const mocks = vi.hoisted(() => ({ storeSessionRequestPhaseSnapshot: vi.fn(async () => {}), storeSessionResponsePhaseSnapshot: vi.fn(async () => {}), getVersionedBindingCapabilityState: vi.fn(() => "available"), + ensureVersionedBindingCapability: vi.fn(async () => "available"), + getSessionBindingSnapshot: vi.fn(async (sessionId: string, keyId: number) => ({ + status: "ok", + legacyFallbackAllowed: false, + source: "existing", + snapshot: { sessionId, keyId, providerId: null, generation: "g-test" }, + })), + acquireSessionDiscoveryLease: vi.fn(async () => ({ + status: "acquired", + ownerToken: "lease-test", + legacyFallbackAllowed: false, + })), + releaseSessionDiscoveryLease: vi.fn(async () => ({ + status: "released", + legacyFallbackAllowed: false, + })), + clearVersionedSessionProvider: vi.fn(async (snapshot: unknown) => ({ + status: "ok", + legacyFallbackAllowed: false, + source: "cleared", + snapshot: { + ...(snapshot as Record), + providerId: null, + generation: "g-cleared", + }, + })), + isWebsocketClientRequest: vi.fn(() => false), })); vi.mock("@/lib/logger", () => ({ @@ -69,6 +96,11 @@ vi.mock("@/lib/provider-endpoints/endpoint-selector", () => ({ getEndpointFilterStats: mocks.getEndpointFilterStats, })); +vi.mock("@/app/v1/_lib/responses-ws/eligibility", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, isWebsocketClientRequest: mocks.isWebsocketClientRequest }; +}); + vi.mock("@/lib/endpoint-circuit-breaker", () => ({ recordEndpointSuccess: mocks.recordEndpointSuccess, recordEndpointFailure: mocks.recordEndpointFailure, @@ -96,6 +128,11 @@ vi.mock("@/lib/rate-limit/service", () => ({ vi.mock("@/lib/session-manager", () => ({ SessionManager: { getVersionedBindingCapabilityState: mocks.getVersionedBindingCapabilityState, + ensureVersionedBindingCapability: mocks.ensureVersionedBindingCapability, + getSessionBindingSnapshot: mocks.getSessionBindingSnapshot, + acquireSessionDiscoveryLease: mocks.acquireSessionDiscoveryLease, + releaseSessionDiscoveryLease: mocks.releaseSessionDiscoveryLease, + clearVersionedSessionProvider: mocks.clearVersionedSessionProvider, updateSessionBindingSmart: mocks.updateSessionBindingSmart, updateSessionProvider: mocks.updateSessionProvider, clearSessionProvider: mocks.clearSessionProvider, @@ -131,8 +168,10 @@ import { import { ProxyForwarder } from "@/app/v1/_lib/proxy/forwarder"; import { ModelRedirector } from "@/app/v1/_lib/proxy/model-redirector"; import { ProxySession } from "@/app/v1/_lib/proxy/session"; +import { DbPoolAdmissionError } from "@/drizzle/admitted-client"; import { logger } from "@/lib/logger"; import type { Provider } from "@/types/provider"; +import type { SystemSettings } from "@/types/system-config"; type AttemptRuntime = { clearResponseTimeout?: () => void; @@ -230,6 +269,8 @@ function createSession(clientAbortSignal: AbortSignal | null = null): ProxySessi provider: null, messageContext: null, sessionId: "sess-hedge", + streamingHedgeDisabled: false, + sessionBindingAllowed: true, requestSequence: 1, originalFormat: "claude", providerType: null, @@ -351,8 +392,124 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { tracked: true, referenced: true, }); + mocks.ensureVersionedBindingCapability.mockResolvedValue("available"); + mocks.getSessionBindingSnapshot.mockImplementation( + async (sessionId: string, keyId: number) => ({ + status: "ok", + legacyFallbackAllowed: false, + source: "existing", + snapshot: { sessionId, keyId, providerId: null, generation: "g-test" }, + }) + ); + mocks.acquireSessionDiscoveryLease.mockResolvedValue({ + status: "acquired", + ownerToken: "lease-test", + legacyFallbackAllowed: false, + }); + mocks.releaseSessionDiscoveryLease.mockResolvedValue({ + status: "released", + legacyFallbackAllowed: false, + }); + mocks.clearVersionedSessionProvider.mockImplementation(async (snapshot: unknown) => ({ + status: "ok", + legacyFallbackAllowed: false, + source: "cleared", + snapshot: { + ...(snapshot as Record), + providerId: null, + generation: "g-cleared", + }, + })); + mocks.categorizeErrorAsync.mockResolvedValue(ProxyErrorCategory.PROVIDER_ERROR); + mocks.isWebsocketClientRequest.mockReturnValue(false); + }); + + test("Discovery actively probes an unknown binding capability before acquiring its lease", async () => { + const provider = createProvider({ id: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 20 }, + apiKey: null, + } as typeof session.authState; + session.setProvider(provider); + mocks.getVersionedBindingCapabilityState.mockReturnValueOnce("unknown"); + mocks.ensureVersionedBindingCapability.mockResolvedValueOnce("available"); + + const prepareStreamingDiscovery = ( + ProxyForwarder as unknown as { + prepareStreamingDiscovery: ( + session: ProxySession, + settings: SystemSettings, + requestStartedAt: number + ) => Promise; + } + ).prepareStreamingDiscovery; + const prepared = await prepareStreamingDiscovery( + session, + { + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 50, + stickySlaMs: 50, + racingTotalTimeoutMs: 200, + stickyTimeoutCooldownMs: 300_000, + } as SystemSettings, + Date.now() + ); + + expect(prepared).not.toBeNull(); + expect(mocks.ensureVersionedBindingCapability).toHaveBeenCalledTimes(1); + expect(mocks.getSessionBindingSnapshot).toHaveBeenCalledWith("sess-hedge", 20); + expect(mocks.acquireSessionDiscoveryLease).toHaveBeenCalledTimes(1); }); + test.each(["unknown", "unavailable"] as const)( + "Discovery fails closed when the binding capability probe returns %s", + async (capabilityState) => { + const provider = createProvider({ id: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 21 }, + apiKey: null, + } as typeof session.authState; + session.setProvider(provider); + mocks.ensureVersionedBindingCapability.mockResolvedValueOnce(capabilityState); + + const prepareStreamingDiscovery = ( + ProxyForwarder as unknown as { + prepareStreamingDiscovery: ( + session: ProxySession, + settings: SystemSettings, + requestStartedAt: number + ) => Promise; + } + ).prepareStreamingDiscovery; + const prepared = await prepareStreamingDiscovery( + session, + { + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 50, + stickySlaMs: 50, + racingTotalTimeoutMs: 200, + stickyTimeoutCooldownMs: 300_000, + } as SystemSettings, + Date.now() + ); + + expect(prepared).toBeNull(); + expect(mocks.ensureVersionedBindingCapability).toHaveBeenCalledTimes(1); + expect(mocks.getSessionBindingSnapshot).not.toHaveBeenCalled(); + expect(mocks.acquireSessionDiscoveryLease).not.toHaveBeenCalled(); + } + ); + test("shadow session redirect should not overwrite initial provider redirect and winner should keep its own redirect", () => { const requestedModel = "claude-haiku-4-5-20251001"; const fireworksRedirect = "accounts/fireworks/routers/kimi-k2p5-turbo"; @@ -2080,6 +2237,12 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { const high = createProvider({ id: 1, name: "high", priority: 1 }); const low = createProvider({ id: 2, name: "low", priority: 10 }); const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 1 }, + apiKey: null, + } as typeof session.authState; session.setProvider(high); mocks.getCachedSystemSettings.mockResolvedValue({ discoveryEnabled: true, @@ -2156,6 +2319,941 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { } }); + test("Discovery lease conflict forces a single upstream and forbids binding writes", async () => { + const provider = createProvider({ id: 1, firstByteTimeoutStreamingMs: 100 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 7 }, + apiKey: null, + } as typeof session.authState; + session.setProvider(provider); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + racingTotalTimeoutMs: 500, + }); + mocks.acquireSessionDiscoveryLease.mockResolvedValueOnce({ + status: "conflict", + reason: "lease_held", + legacyFallbackAllowed: false, + }); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockResolvedValueOnce( + new Response('data: {"type":"message_stop"}\n\n', { + status: 200, + headers: { "content-type": "text/event-stream" }, + }) + ); + + const response = await ProxyForwarder.send(session); + expect(response.status).toBe(200); + expect(doForward).toHaveBeenCalledTimes(1); + expect(session.isStreamingHedgeDisabled()).toBe(true); + expect(session.isSessionBindingAllowed()).toBe(false); + expect(mocks.pickDiscoveryProviders).not.toHaveBeenCalled(); + expect(mocks.releaseSessionDiscoveryLease).not.toHaveBeenCalled(); + expect(mocks.getCachedSystemSettings).toHaveBeenCalledTimes(1); + }); + + test("foreign binding state fails closed before Discovery acquires a lease", async () => { + const provider = createProvider({ id: 1, firstByteTimeoutStreamingMs: 0 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 8 }, + apiKey: null, + } as typeof session.authState; + session.setProvider(provider); + mocks.getCachedSystemSettings.mockResolvedValue({ discoveryEnabled: true }); + mocks.getSessionBindingSnapshot.mockResolvedValueOnce({ + status: "conflict", + reason: "legacy_owner_mismatch", + legacyFallbackAllowed: false, + }); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockResolvedValueOnce( + new Response('data: {"type":"message_stop"}\n\n', { + status: 200, + headers: { "content-type": "text/event-stream" }, + }) + ); + + await ProxyForwarder.send(session); + expect(doForward).toHaveBeenCalledTimes(1); + expect(session.isSessionBindingAllowed()).toBe(false); + expect(mocks.acquireSessionDiscoveryLease).not.toHaveBeenCalled(); + expect(mocks.pickDiscoveryProviders).not.toHaveBeenCalled(); + }); + + test("Sticky fallback stays held while timeout CAS and the next wave are being prepared", async () => { + vi.useFakeTimers(); + try { + const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); + const normal = createProvider({ id: 2, name: "normal", priority: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 9 }, + apiKey: null, + } as typeof session.authState; + session.request.message.messages = [ + { role: "user", content: "first" }, + { role: "user", content: "second" }, + ]; + session.setProvider(sticky); + session.setSessionBindingSnapshot({ + sessionId: session.sessionId!, + keyId: 9, + providerId: sticky.id, + generation: "g-sticky", + }); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 2, + discoverySlaMs: 50, + stickySlaMs: 10, + racingTotalTimeoutMs: 200, + stickyTimeoutCooldownMs: 300_000, + }); + mocks.pickDiscoveryProviders.mockResolvedValueOnce([normal]); + + let resolveClear!: (value: unknown) => void; + mocks.clearVersionedSessionProvider.mockReturnValueOnce( + new Promise((resolve) => { + resolveClear = resolve; + }) + ); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockResolvedValueOnce( + new Response( + new ReadableStream({ + start(controller) { + setTimeout(() => { + controller.enqueue( + new TextEncoder().encode( + 'data: {"type":"content_block_delta","delta":{"text":"sticky"}}\n\n' + ) + ); + controller.close(); + }, 15); + }, + }), + { headers: { "content-type": "text/event-stream" } } + ) + ); + doForward.mockResolvedValueOnce( + new Response( + new ReadableStream({ + start(controller) { + setTimeout(() => { + controller.enqueue( + new TextEncoder().encode( + 'data: {"type":"content_block_delta","delta":{"text":"normal"}}\n\n' + ) + ); + controller.close(); + }, 5); + }, + }), + { headers: { "content-type": "text/event-stream" } } + ) + ); + + let settledEarly = false; + const responsePromise = ProxyForwarder.send(session).then((response) => { + settledEarly = true; + return response; + }); + await vi.advanceTimersByTimeAsync(20); + expect(settledEarly).toBe(false); + + resolveClear({ + status: "ok", + legacyFallbackAllowed: false, + source: "cleared", + snapshot: { + sessionId: session.sessionId!, + keyId: 9, + providerId: null, + generation: "g-cleared", + }, + }); + await vi.advanceTimersByTimeAsync(10); + const response = await responsePromise; + expect(await response.text()).toContain('"normal"'); + expect(session.provider?.id).toBe(normal.id); + } finally { + vi.useRealTimers(); + } + }); + + test("Sticky probing does not consume a configured Discovery round", async () => { + vi.useFakeTimers(); + try { + const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); + const roundOne = createProvider({ id: 2, name: "round-one", priority: 1 }); + const roundTwo = createProvider({ id: 3, name: "round-two", priority: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 19 }, + apiKey: null, + } as typeof session.authState; + session.request.message.messages = [ + { role: "user", content: "first" }, + { role: "user", content: "second" }, + ]; + session.setProvider(sticky); + session.setSessionBindingSnapshot({ + sessionId: session.sessionId!, + keyId: 19, + providerId: sticky.id, + generation: "g-sticky-rounds", + }); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 2, + discoverySlaMs: 20, + stickySlaMs: 10, + racingTotalTimeoutMs: 100, + stickyTimeoutCooldownMs: 300_000, + }); + mocks.pickDiscoveryProviders + .mockResolvedValueOnce([roundOne]) + .mockResolvedValueOnce([roundTwo]); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockImplementation(async (attemptSession) => { + const providerId = (attemptSession as ProxySession).provider?.id; + if (providerId !== roundTwo.id) { + return new Response(new ReadableStream(), { + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response( + new ReadableStream({ + start(controller) { + setTimeout(() => { + controller.enqueue( + new TextEncoder().encode( + 'data: {"type":"content_block_delta","delta":{"text":"round-two"}}\n\n' + ) + ); + controller.close(); + }, 5); + }, + }), + { headers: { "content-type": "text/event-stream" } } + ); + }); + + const responsePromise = ProxyForwarder.send(session); + await vi.advanceTimersByTimeAsync(10); + await vi.advanceTimersByTimeAsync(20); + await vi.advanceTimersByTimeAsync(5); + + const response = await responsePromise; + expect(await response.text()).toContain('"round-two"'); + expect(session.provider?.id).toBe(roundTwo.id); + expect(mocks.pickDiscoveryProviders).toHaveBeenCalledTimes(2); + expect(doForward).toHaveBeenCalledTimes(3); + } finally { + vi.useRealTimers(); + } + }); + + test("Sticky timeout still starts one normal wave when maxDiscoveryRounds is one", async () => { + vi.useFakeTimers(); + try { + const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); + const normal = createProvider({ id: 2, name: "normal", priority: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 21 }, + apiKey: null, + } as typeof session.authState; + session.request.message.messages = [ + { role: "user", content: "first" }, + { role: "user", content: "second" }, + ]; + session.setProvider(sticky); + session.setSessionBindingSnapshot({ + sessionId: session.sessionId!, + keyId: 21, + providerId: sticky.id, + generation: "g-sticky-one-round", + }); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 20, + stickySlaMs: 10, + racingTotalTimeoutMs: 100, + stickyTimeoutCooldownMs: 300_000, + }); + mocks.pickDiscoveryProviders.mockResolvedValueOnce([normal]); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockImplementation(async (attemptSession) => { + if ((attemptSession as ProxySession).provider?.id === sticky.id) { + return new Response(new ReadableStream(), { + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response('data: {"type":"content_block_delta","delta":{"text":"normal"}}\n\n', { + headers: { "content-type": "text/event-stream" }, + }); + }); + + const responsePromise = ProxyForwarder.send(session); + await vi.advanceTimersByTimeAsync(10); + const response = await responsePromise; + + expect(await response.text()).toContain('"normal"'); + expect(mocks.pickDiscoveryProviders).toHaveBeenCalledWith( + expect.anything(), + 1, + expect.arrayContaining([sticky.id]) + ); + expect(doForward).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + + test("an explicit Sticky failure starts Discovery round one at full concurrency", async () => { + const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); + const normal = createProvider({ id: 2, name: "normal", priority: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 22 }, + apiKey: null, + } as typeof session.authState; + session.request.message.messages = [ + { role: "user", content: "first" }, + { role: "user", content: "second" }, + ]; + session.setProvider(sticky); + session.setSessionBindingSnapshot({ + sessionId: session.sessionId!, + keyId: 22, + providerId: sticky.id, + generation: "g-sticky-failure", + }); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 50, + stickySlaMs: 50, + racingTotalTimeoutMs: 200, + stickyTimeoutCooldownMs: 300_000, + }); + mocks.pickDiscoveryProviders.mockResolvedValueOnce([normal]); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockRejectedValueOnce(new Error("Sticky upstream failed")).mockResolvedValueOnce( + new Response('data: {"type":"content_block_delta","delta":{"text":"normal"}}\n\n', { + headers: { "content-type": "text/event-stream" }, + }) + ); + + const response = await ProxyForwarder.send(session); + expect(await response.text()).toContain('"normal"'); + expect(mocks.pickDiscoveryProviders).toHaveBeenCalledWith( + expect.anything(), + 2, + expect.arrayContaining([sticky.id]) + ); + expect(doForward).toHaveBeenCalledTimes(2); + expect(mocks.clearVersionedSessionProvider).toHaveBeenCalledWith( + expect.objectContaining({ providerId: sticky.id, generation: "g-sticky-failure" }), + sticky.id, + 0 + ); + }); + + test("Discovery eligibility excludes WebSocket-tunneled requests", async () => { + const provider = createProvider({ id: 1, firstByteTimeoutStreamingMs: 0 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 23 }, + apiKey: null, + } as typeof session.authState; + session.setProvider(provider); + mocks.getCachedSystemSettings.mockResolvedValue({ discoveryEnabled: true }); + mocks.isWebsocketClientRequest.mockReturnValueOnce(true); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockResolvedValueOnce( + new Response('data: {"type":"message_stop"}\n\n', { + status: 200, + headers: { "content-type": "text/event-stream" }, + }) + ); + + await ProxyForwarder.send(session); + expect(doForward).toHaveBeenCalledTimes(1); + expect(mocks.acquireSessionDiscoveryLease).not.toHaveBeenCalled(); + expect(mocks.pickDiscoveryProviders).not.toHaveBeenCalled(); + }); + + test("Discovery stops immediately on local database admission overload", async () => { + const provider = createProvider({ id: 1 }); + const alternative = createProvider({ id: 2 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 24 }, + apiKey: null, + } as typeof session.authState; + session.setProvider(provider); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 50, + stickySlaMs: 50, + racingTotalTimeoutMs: 200, + }); + mocks.pickDiscoveryProviders.mockResolvedValueOnce([alternative]); + mocks.categorizeErrorAsync.mockResolvedValueOnce(ProxyErrorCategory.LOCAL_OVERLOAD); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + const overload = new DbPoolAdmissionError("data", 32); + doForward.mockRejectedValueOnce(overload); + + await expect(ProxyForwarder.send(session)).rejects.toBe(overload); + expect(doForward).toHaveBeenCalledTimes(1); + expect(mocks.recordFailure).not.toHaveBeenCalled(); + expect(mocks.releaseSessionDiscoveryLease).toHaveBeenCalledTimes(1); + }); + + test("Discovery total deadline is not blocked by a stalled candidate selector", async () => { + vi.useFakeTimers(); + try { + const provider = createProvider({ id: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 25 }, + apiKey: null, + } as typeof session.authState; + session.setProvider(provider); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 20, + stickySlaMs: 20, + racingTotalTimeoutMs: 50, + }); + mocks.pickDiscoveryProviders.mockReturnValueOnce(new Promise(() => {})); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockResolvedValueOnce( + new Response(new ReadableStream(), { + headers: { "content-type": "text/event-stream" }, + }) + ); + + const responsePromise = ProxyForwarder.send(session); + const observedError = responsePromise.catch((error) => error); + await vi.advanceTimersByTimeAsync(50); + expect(await observedError).toBeInstanceOf(UpstreamProxyError); + expect(mocks.releaseSessionDiscoveryLease).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } + }); + + test("a fallback failure waits for the reserved wave before advancing another round", async () => { + vi.useFakeTimers(); + try { + const fallback = createProvider({ id: 1, name: "fallback", priority: 1 }); + const firstRoundLoser = createProvider({ id: 2, name: "first-round-loser", priority: 1 }); + const nextRoundWinner = createProvider({ id: 3, name: "next-round-winner", priority: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 26 }, + apiKey: null, + } as typeof session.authState; + session.setProvider(fallback); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 3, + discoverySlaMs: 10, + stickySlaMs: 10, + racingTotalTimeoutMs: 100, + }); + + const reservedWave = Promise.withResolvers(); + mocks.pickDiscoveryProviders + .mockResolvedValueOnce([firstRoundLoser]) + .mockReturnValueOnce(reservedWave.promise) + .mockResolvedValueOnce([ + createProvider({ id: 4, name: "unexpected-extra-round", priority: 1 }), + ]); + + const fallbackFailure = Promise.withResolvers(); + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockImplementation(async (attemptSession) => { + const providerId = (attemptSession as ProxySession).provider?.id; + if (providerId === fallback.id) return fallbackFailure.promise; + if (providerId === nextRoundWinner.id) { + return new Response( + 'data: {"type":"content_block_delta","delta":{"text":"winner"}}\n\n', + { headers: { "content-type": "text/event-stream" } } + ); + } + return new Response(new ReadableStream(), { + headers: { "content-type": "text/event-stream" }, + }); + }); + + const responsePromise = ProxyForwarder.send(session); + await vi.advanceTimersByTimeAsync(10); + expect(mocks.pickDiscoveryProviders).toHaveBeenCalledTimes(2); + + fallbackFailure.reject(new Error("fallback failed during reserved wave")); + await vi.advanceTimersByTimeAsync(0); + expect(mocks.pickDiscoveryProviders).toHaveBeenCalledTimes(2); + + reservedWave.resolve([nextRoundWinner]); + await vi.advanceTimersByTimeAsync(0); + const response = await responsePromise; + expect(await response.text()).toContain('"winner"'); + expect(mocks.pickDiscoveryProviders).toHaveBeenCalledTimes(2); + expect(doForward).toHaveBeenCalledTimes(3); + } finally { + vi.useRealTimers(); + } + }); + + test("Discovery does not immediately reselect a Provider whose launch setup failed", async () => { + const initial = createProvider({ id: 1, name: "initial" }); + const alternative = createProvider({ id: 2, name: "alternative" }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 20 }, + apiKey: null, + } as typeof session.authState; + session.setProvider(initial); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 50, + stickySlaMs: 50, + racingTotalTimeoutMs: 200, + stickyTimeoutCooldownMs: 300_000, + }); + mocks.pickDiscoveryProviders.mockImplementationOnce( + async (_session: ProxySession, _count: number, excludedIds: number[]) => { + expect(excludedIds).toContain(initial.id); + return [alternative]; + } + ); + + const endpointResolver = vi.spyOn( + ProxyForwarder as unknown as { + resolveStreamingHedgeEndpoint: ( + session: ProxySession, + provider: Provider + ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + }, + "resolveStreamingHedgeEndpoint" + ); + endpointResolver + .mockRejectedValueOnce(new Error("initial endpoint setup failed")) + .mockResolvedValue({ + endpointId: null, + baseUrl: alternative.url, + endpointUrl: alternative.url, + }); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockResolvedValueOnce( + new Response('data: {"type":"content_block_delta","delta":{"text":"alternative"}}\n\n', { + headers: { "content-type": "text/event-stream" }, + }) + ); + + try { + const response = await ProxyForwarder.send(session); + expect(await response.text()).toContain('"alternative"'); + expect(doForward).toHaveBeenCalledTimes(1); + expect(session.provider?.id).toBe(alternative.id); + } finally { + endpointResolver.mockRestore(); + } + }); + + test("Discovery transfers the Provider session ref when a rectifier retries the same Provider", async () => { + const initial = createProvider({ id: 1, name: "initial", limitConcurrentSessions: 1 }); + const alternative = createProvider({ id: 2, name: "alternative", limitConcurrentSessions: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 22 }, + apiKey: null, + } as typeof session.authState; + setProviderWithSessionRef(session, initial); + withThinkingBlocks(session); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 100, + stickySlaMs: 100, + racingTotalTimeoutMs: 500, + stickyTimeoutCooldownMs: 300_000, + enableThinkingSignatureRectifier: true, + }); + mocks.pickDiscoveryProviders.mockResolvedValueOnce([alternative]); + + const signatureError = new UpstreamProxyError("Invalid `signature` in `thinking` block", 400, { + body: '{"error":"invalid_signature"}', + providerId: initial.id, + providerName: initial.name, + }); + let initialAttempts = 0; + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockImplementation(async (attemptSession) => { + const runtime = attemptSession as ProxySession & AttemptRuntime; + if (runtime.provider?.id === initial.id) { + initialAttempts += 1; + if (initialAttempts === 1) throw signatureError; + + const body = runtime.request.message as { + messages: Array<{ content: Array> }>; + }; + expect(body.messages[0].content.some((block) => "signature" in block)).toBe(false); + return new Response( + 'data: {"type":"content_block_delta","delta":{"text":"rectified"}}\n\n', + { headers: { "content-type": "text/event-stream" } } + ); + } + + return new Response(new ReadableStream(), { + headers: { "content-type": "text/event-stream" }, + }); + }); + + const response = await ProxyForwarder.send(session); + expect(await response.text()).toContain('"rectified"'); + + const initialAdmissionCalls = mocks.checkAndTrackProviderSession.mock.calls.filter( + ([providerId]) => providerId === initial.id + ); + const initialReleaseCalls = mocks.releaseProviderSession.mock.calls.filter( + ([providerId]) => providerId === initial.id + ); + expect(initialAttempts).toBe(2); + expect(initialAdmissionCalls).toHaveLength(0); + expect(initialReleaseCalls).toHaveLength(0); + expect(session.hasProviderSessionRef(initial.id)).toBe(true); + }); + + test("Discovery releases a transferred Provider session ref exactly once when rectifier retry setup fails", async () => { + const provider = createProvider({ id: 1, name: "initial", limitConcurrentSessions: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 23 }, + apiKey: null, + } as typeof session.authState; + setProviderWithSessionRef(session, provider); + withThinkingBlocks(session); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 100, + stickySlaMs: 100, + racingTotalTimeoutMs: 500, + stickyTimeoutCooldownMs: 300_000, + enableThinkingSignatureRectifier: true, + }); + mocks.pickDiscoveryProviders.mockResolvedValueOnce([]); + + const endpointResolver = vi.spyOn( + ProxyForwarder as unknown as { + resolveStreamingHedgeEndpoint: ( + session: ProxySession, + provider: Provider + ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + }, + "resolveStreamingHedgeEndpoint" + ); + endpointResolver + .mockResolvedValueOnce({ + endpointId: null, + baseUrl: provider.url, + endpointUrl: provider.url, + }) + .mockRejectedValueOnce(new Error("rectifier retry endpoint setup failed")); + + const signatureError = new UpstreamProxyError("Invalid `signature` in `thinking` block", 400, { + body: '{"error":"invalid_signature"}', + providerId: provider.id, + providerName: provider.name, + }); + vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ).mockRejectedValueOnce(signatureError); + + try { + await expect(ProxyForwarder.send(session)).rejects.toBeInstanceOf(Error); + const providerAdmissionCalls = mocks.checkAndTrackProviderSession.mock.calls.filter( + ([providerId]) => providerId === provider.id + ); + const providerReleaseCalls = mocks.releaseProviderSession.mock.calls.filter( + ([providerId]) => providerId === provider.id + ); + expect(providerAdmissionCalls).toHaveLength(0); + expect(providerReleaseCalls).toHaveLength(1); + expect(session.hasProviderSessionRef(provider.id)).toBe(false); + } finally { + endpointResolver.mockRestore(); + } + }); + + test("a candidate delayed in launch setup is rolled back after another attempt wins", async () => { + vi.useFakeTimers(); + try { + const initial = createProvider({ id: 1, name: "initial" }); + const delayed = createProvider({ id: 2, name: "delayed", limitConcurrentSessions: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 10 }, + apiKey: null, + } as typeof session.authState; + setProviderWithSessionRef(session, initial); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 50, + stickySlaMs: 50, + racingTotalTimeoutMs: 200, + }); + mocks.pickDiscoveryProviders.mockResolvedValueOnce([delayed]); + + let resolveAdmission!: (value: unknown) => void; + mocks.checkAndTrackProviderSession.mockReturnValueOnce( + new Promise((resolve) => { + resolveAdmission = resolve; + }) + ); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockResolvedValueOnce( + new Response( + new ReadableStream({ + start(controller) { + setTimeout(() => { + controller.enqueue( + new TextEncoder().encode( + 'data: {"type":"content_block_delta","delta":{"text":"winner"}}\n\n' + ) + ); + controller.close(); + }, 5); + }, + }), + { headers: { "content-type": "text/event-stream" } } + ) + ); + + const responsePromise = ProxyForwarder.send(session); + await vi.advanceTimersByTimeAsync(10); + resolveAdmission({ allowed: true, count: 1, tracked: true, referenced: true }); + await vi.advanceTimersByTimeAsync(1); + const response = await responsePromise; + expect(await response.text()).toContain('"winner"'); + expect(doForward).toHaveBeenCalledTimes(1); + expect(mocks.releaseProviderSession).toHaveBeenCalledWith(delayed.id, session.sessionId); + } finally { + vi.useRealTimers(); + } + }); + + test("Discovery client abort preserves the captured binding and releases its lease", async () => { + const clientAbort = new AbortController(); + const provider = createProvider({ id: 1 }); + const session = createSession(clientAbort.signal); + session.authState = { + success: true, + user: null, + key: { id: 11 }, + apiKey: null, + } as typeof session.authState; + setProviderWithSessionRef(session, provider); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 100, + stickySlaMs: 100, + racingTotalTimeoutMs: 500, + }); + mocks.pickDiscoveryProviders.mockResolvedValueOnce([]); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockImplementationOnce( + async (_attemptSession, _provider, _baseUrl, _audit, _count, _stream, signal) => + await new Promise((_resolve, reject) => { + signal?.addEventListener("abort", () => reject(signal.reason), { once: true }); + }) + ); + + const responsePromise = ProxyForwarder.send(session); + clientAbort.abort(new Error("client disconnected")); + await expect(responsePromise).rejects.toMatchObject({ statusCode: 499 }); + expect(mocks.clearVersionedSessionProvider).not.toHaveBeenCalled(); + expect(mocks.clearSessionProviders).not.toHaveBeenCalled(); + expect(mocks.releaseSessionDiscoveryLease).toHaveBeenCalledWith( + session.sessionId, + 11, + "lease-test" + ); + }); + + test("Discovery preserves binding state for a non-retryable client error", async () => { + const provider = createProvider({ id: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 12 }, + apiKey: null, + } as typeof session.authState; + setProviderWithSessionRef(session, provider); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 100, + stickySlaMs: 100, + racingTotalTimeoutMs: 500, + }); + mocks.pickDiscoveryProviders.mockResolvedValueOnce([]); + mocks.categorizeErrorAsync.mockResolvedValueOnce(ProxyErrorCategory.NON_RETRYABLE_CLIENT_ERROR); + const clientError = new UpstreamProxyError("invalid request", 400); + vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ).mockRejectedValueOnce(clientError); + + await expect(ProxyForwarder.send(session)).rejects.toBe(clientError); + expect(mocks.clearVersionedSessionProvider).not.toHaveBeenCalled(); + expect(mocks.clearSessionProviders).not.toHaveBeenCalled(); + expect(mocks.releaseSessionDiscoveryLease).toHaveBeenCalled(); + }); + test("removes streaming hedge client abort listener after winner response is returned", async () => { const clientAbortController = new AbortController(); const addSpy = vi.spyOn(clientAbortController.signal, "addEventListener"); diff --git a/tests/unit/proxy/proxy-forwarder-provider-session-release.test.ts b/tests/unit/proxy/proxy-forwarder-provider-session-release.test.ts index 0fa7ebafa..89d1bbb9c 100644 --- a/tests/unit/proxy/proxy-forwarder-provider-session-release.test.ts +++ b/tests/unit/proxy/proxy-forwarder-provider-session-release.test.ts @@ -22,6 +22,20 @@ describe("ProxyForwarder provider failure session release", () => { mocks.releaseProviderSession.mockClear(); }); + it("tracks baseline ownership independently for consecutive refs to one Provider", async () => { + const { ProxySession } = await import("@/app/v1/_lib/proxy/session"); + const session = Object.create(ProxySession.prototype) as ProxySession; + + session.recordProviderSessionRef(42, { retainOnSuccess: true }); + session.recordProviderSessionRef(42, { retainOnSuccess: false }); + + expect(session.shouldRetainProviderSessionRefOnSuccess(42)).toBe(true); + expect(session.consumeProviderSessionRef(42)).toBe(true); + expect(session.shouldRetainProviderSessionRefOnSuccess(42)).toBe(false); + expect(session.consumeProviderSessionRef(42)).toBe(true); + expect(session.hasProviderSessionRef(42)).toBe(false); + }); + it("标记供应商失败时仅释放本请求已获取的 provider session ref", async () => { const { ProxyForwarder } = await import("@/app/v1/_lib/proxy/forwarder"); const forwarderInternals = ProxyForwarder as unknown as { diff --git a/tests/unit/proxy/response-handler-client-abort-drain.test.ts b/tests/unit/proxy/response-handler-client-abort-drain.test.ts index ee817d348..ded21ba4b 100644 --- a/tests/unit/proxy/response-handler-client-abort-drain.test.ts +++ b/tests/unit/proxy/response-handler-client-abort-drain.test.ts @@ -104,6 +104,7 @@ vi.mock("@/lib/rate-limit", () => ({ trackUserDailyCost: vi.fn(), decrementLeaseBudget: vi.fn(), settleLeaseBudgets: vi.fn(), + releaseProviderSession: vi.fn(), }, })); @@ -114,6 +115,17 @@ vi.mock("@/lib/redis/live-chain-store", () => ({ vi.mock("@/lib/session-manager", () => ({ SessionManager: { clearSessionProvider: vi.fn(), + clearVersionedSessionProvider: vi.fn(), + compareAndSetSessionProvider: vi.fn(), + getSessionBindingSnapshot: vi.fn(), + renewSessionDiscoveryLease: vi.fn(async () => ({ + status: "renewed", + legacyFallbackAllowed: false, + })), + releaseSessionDiscoveryLease: vi.fn(async () => ({ + status: "released", + legacyFallbackAllowed: false, + })), extractCodexPromptCacheKey: vi.fn(), storeSessionResponse: vi.fn(async () => undefined), storeSessionRequestPhaseSnapshot: vi.fn(), @@ -2074,6 +2086,69 @@ describe("ProxyResponseHandler stream client abort finalization", () => { ); }); + it.each([ + { bindingIntent: "create" as const, providerId: null }, + { bindingIntent: "renew" as const, providerId: 1 }, + ])( + "preserves binding state for a client-aborted Discovery $bindingIntent stream", + async ({ bindingIntent, providerId }) => { + const controller = new AbortController(); + controller.abort(); + const session = createSession(controller.signal); + Object.assign(session, { sessionId: `session-client-abort-${bindingIntent}` }); + session.recordProviderSessionRef(1); + vi.mocked(SessionManager.extractCodexPromptCacheKey).mockReturnValue( + "client-abort-cache-key" + ); + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "avemujica-responses", + providerPriority: 1, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: bindingIntent === "create", + endpointId: 42, + endpointUrl: "https://api.test.invalid/v1", + upstreamStatusCode: 200, + bindingIntent, + bindingSnapshot: { + sessionId: `session-client-abort-${bindingIntent}`, + keyId: 2, + providerId, + generation: `${bindingIntent}-generation`, + }, + requiresCompletionMarker: true, + discoveryLease: { + sessionId: `session-client-abort-${bindingIntent}`, + keyId: 2, + ownerToken: `client-abort-${bindingIntent}-owner`, + ttlSeconds: 30, + }, + providerSessionRefOwned: true, + }); + + await ProxyResponseHandler.dispatch(session, createCompletedThenErroredResponsesSse()); + await drainAsyncTasks(); + + expect(SessionManager.clearVersionedSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.clearSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.updateSessionBindingSmart).not.toHaveBeenCalled(); + expect(SessionManager.updateSessionWithCodexCacheKey).not.toHaveBeenCalled(); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledWith( + `session-client-abort-${bindingIntent}`, + 2, + `client-abort-${bindingIntent}-owner` + ); + expect(RateLimitService.releaseProviderSession).toHaveBeenCalledWith( + 1, + `session-client-abort-${bindingIntent}` + ); + } + ); + it("keeps a genuinely aborted upstream responses stream as 499", async () => { const controller = new AbortController(); controller.abort(); @@ -3184,6 +3259,45 @@ describe("ProxyResponseHandler stream client abort finalization", () => { ); }); + it("does not mutate non-stream bindings when the routing mode forbids it", async () => { + const controller = new AbortController(); + controller.abort(); + const session = createSession(controller.signal); + Object.assign(session, { + sessionId: "lease-conflict-non-stream", + isSessionBindingAllowed: () => false, + }); + vi.mocked(SessionManager.extractCodexPromptCacheKey).mockReturnValue("blocked-cache-key"); + const response = new Response('{"id":"resp_lease_conflict"}', { + status: 200, + headers: { "content-type": "application/json" }, + }); + + await ProxyResponseHandler.dispatch(session, response); + await drainAsyncTasks(); + + expect(SessionManager.clearSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.updateSessionWithCodexCacheKey).not.toHaveBeenCalled(); + }); + + it("does not create a non-stream Codex cache binding when binding is disabled", async () => { + const session = createSession(new AbortController().signal); + Object.assign(session, { + sessionId: "lease-conflict-non-stream-success", + isSessionBindingAllowed: () => false, + }); + vi.mocked(SessionManager.extractCodexPromptCacheKey).mockReturnValue("blocked-cache-key"); + const response = new Response('{"id":"resp_lease_conflict"}', { + status: 200, + headers: { "content-type": "application/json" }, + }); + + await ProxyResponseHandler.dispatch(session, response); + await drainAsyncTasks(); + + expect(SessionManager.updateSessionWithCodexCacheKey).not.toHaveBeenCalled(); + }); + it("publishes a successful stream Codex cache binding only after durable acknowledgement", async () => { const durableAck = createDeferred(); const cacheBinding = createDeferred(); @@ -3243,6 +3357,122 @@ describe("ProxyResponseHandler stream client abort finalization", () => { } }); + it("publishes a Discovery Codex cache key only after the primary generation CAS succeeds", async () => { + const order: string[] = []; + vi.mocked(SessionManager.extractCodexPromptCacheKey).mockReturnValueOnce( + "discovery-stream-cache-key" + ); + vi.mocked(SessionManager.compareAndSetSessionProvider).mockImplementationOnce(async () => { + order.push("primary-cas"); + return { + status: "ok", + source: "updated", + snapshot: { + sessionId: "stream-discovery-cache-binding", + keyId: 2, + providerId: 1, + generation: "discovery-updated-generation", + }, + legacyFallbackAllowed: false, + }; + }); + vi.mocked(SessionManager.updateSessionWithCodexCacheKey).mockImplementationOnce(async () => { + order.push("aux-cache-binding"); + }); + const session = createSession(new AbortController().signal); + session.sessionId = "stream-discovery-cache-binding"; + session.recordProviderSessionRef(1); + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "avemujica-responses", + providerPriority: 1, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.invalid/v1", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: { + sessionId: "stream-discovery-cache-binding", + keyId: 2, + providerId: null, + generation: "discovery-create-generation", + }, + requiresCompletionMarker: true, + discoveryLease: { + sessionId: "stream-discovery-cache-binding", + keyId: 2, + ownerToken: "discovery-cache-owner", + ttlSeconds: 30, + }, + providerSessionRefOwned: true, + providerSessionRefRetainOnSuccess: true, + }); + + const downstream = await ProxyResponseHandler.dispatch(session, createResponsesSse()); + await downstream.text(); + await drainAsyncTasks(); + + expect(order).toEqual(["primary-cas", "aux-cache-binding"]); + expect(RateLimitService.releaseProviderSession).not.toHaveBeenCalled(); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); + }); + + it("does not publish a Discovery Codex cache key when the primary generation CAS conflicts", async () => { + vi.mocked(SessionManager.extractCodexPromptCacheKey).mockReturnValueOnce( + "conflicted-discovery-cache-key" + ); + vi.mocked(SessionManager.compareAndSetSessionProvider).mockResolvedValueOnce({ + status: "conflict", + reason: "generation_mismatch", + legacyFallbackAllowed: false, + }); + const session = createSession(new AbortController().signal); + session.sessionId = "stream-discovery-cache-conflict"; + session.recordProviderSessionRef(1); + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "avemujica-responses", + providerPriority: 1, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.invalid/v1", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: { + sessionId: "stream-discovery-cache-conflict", + keyId: 2, + providerId: null, + generation: "stale-discovery-generation", + }, + requiresCompletionMarker: true, + discoveryLease: { + sessionId: "stream-discovery-cache-conflict", + keyId: 2, + ownerToken: "conflicted-discovery-owner", + ttlSeconds: 30, + }, + providerSessionRefOwned: true, + }); + + const downstream = await ProxyResponseHandler.dispatch(session, createResponsesSse()); + await downstream.text(); + await drainAsyncTasks(); + + expect(SessionManager.compareAndSetSessionProvider).toHaveBeenCalledOnce(); + expect(SessionManager.updateSessionWithCodexCacheKey).not.toHaveBeenCalled(); + expect(RateLimitService.releaseProviderSession).toHaveBeenCalledWith( + 1, + "stream-discovery-cache-conflict" + ); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); + }); + it("does not publish a stream Codex cache binding for a final non-2xx outcome", async () => { vi.mocked(SessionManager.extractCodexPromptCacheKey).mockReturnValueOnce("stream-cache-key-2"); const session = createSession(new AbortController().signal); @@ -3276,6 +3506,33 @@ describe("ProxyResponseHandler stream client abort finalization", () => { expect(SessionManager.updateSessionWithCodexCacheKey).not.toHaveBeenCalled(); }); + it("does not publish a Codex cache binding for a Discovery fallback winner", async () => { + vi.mocked(SessionManager.extractCodexPromptCacheKey).mockReturnValueOnce( + "fallback-stream-cache-key" + ); + const session = createSession(new AbortController().signal); + session.sessionId = "stream-codex-fallback"; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "avemujica-responses", + providerPriority: 1, + attemptNumber: 2, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: false, + endpointId: 42, + endpointUrl: "https://api.test.invalid/v1", + upstreamStatusCode: 200, + bindingIntent: "none", + }); + + const downstream = await ProxyResponseHandler.dispatch(session, createResponsesSse()); + await downstream.text(); + await drainAsyncTasks(); + + expect(SessionManager.updateSessionWithCodexCacheKey).not.toHaveBeenCalled(); + }); + it("durably finalizes a Gemini non-stream passthrough body-read failure", async () => { const session = createSession(new AbortController().signal, { providerType: "gemini", diff --git a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts index af432f126..038c9378c 100644 --- a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts +++ b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts @@ -77,6 +77,11 @@ vi.mock("@/lib/session-manager", () => ({ updateSessionUsage: vi.fn(), storeSessionResponse: vi.fn(), clearSessionProvider: vi.fn(), + clearVersionedSessionProvider: vi.fn(), + compareAndSetSessionProvider: vi.fn(), + getSessionBindingSnapshot: vi.fn(), + renewSessionDiscoveryLease: vi.fn(), + releaseSessionDiscoveryLease: vi.fn(), extractCodexPromptCacheKey: vi.fn(), updateSessionBindingSmart: vi.fn(), updateSessionProvider: vi.fn(), @@ -90,6 +95,7 @@ vi.mock("@/lib/rate-limit", () => ({ trackUserDailyCost: vi.fn(), decrementLeaseBudget: vi.fn(), settleLeaseBudgets: vi.fn(), + releaseProviderSession: vi.fn(), }, })); @@ -335,6 +341,69 @@ function createSuccessStreamResponse(): Response { }); } +function createSuccessStreamResponseWithCompletion(): Response { + const sseText = + `data: ${JSON.stringify({ type: "content_block_delta", delta: { text: "ok" } })}\n\n` + + `event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n`; + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sseText)); + controller.close(); + }, + }); + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} + +function createMisleadingCompletionTextResponse(): Response { + const sseText = + `event: content_block_delta\ndata: ${JSON.stringify({ + type: "content_block_delta", + delta: { text: "the words message_stop and response.completed are ordinary content" }, + })}\n\n` + + `event: message_delta\ndata: ${JSON.stringify({ + type: "message_delta", + delta: { stop_reason: null }, + })}\n\n`; + return new Response(sseText, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} + +function createControllableSuccessStreamResponse(): { + response: Response; + complete: () => void; +} { + const encoder = new TextEncoder(); + let streamController!: ReadableStreamDefaultController; + const stream = new ReadableStream({ + start(controller) { + streamController = controller; + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ type: "content_block_delta", delta: { text: "ok" } })}\n\n` + ) + ); + }, + }); + return { + response: new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), + complete: () => { + streamController.enqueue( + encoder.encode(`data: ${JSON.stringify({ type: "message_stop" })}\n\n`) + ); + streamController.close(); + }, + }; +} + async function drainAsyncTasks(): Promise { while (asyncTasks.length > 0) { const tasks = asyncTasks.splice(0, asyncTasks.length); @@ -366,6 +435,47 @@ function setupCommonMocks() { vi.mocked(updateMessageRequestDuration).mockResolvedValue(undefined); vi.mocked(SessionManager.storeSessionResponse).mockResolvedValue(undefined); vi.mocked(SessionManager.clearSessionProvider).mockResolvedValue(undefined); + vi.mocked(SessionManager.clearVersionedSessionProvider).mockResolvedValue({ + status: "ok", + source: "cleared", + snapshot: { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "cleared", + }, + legacyFallbackAllowed: false, + }); + vi.mocked(SessionManager.compareAndSetSessionProvider).mockResolvedValue({ + status: "ok", + source: "updated", + snapshot: { + sessionId: "fake-session", + keyId: 456, + providerId: 1, + generation: "updated", + }, + legacyFallbackAllowed: false, + }); + vi.mocked(SessionManager.getSessionBindingSnapshot).mockResolvedValue({ + status: "ok", + source: "existing", + snapshot: { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "fresh", + }, + legacyFallbackAllowed: false, + }); + vi.mocked(SessionManager.renewSessionDiscoveryLease).mockResolvedValue({ + status: "renewed", + legacyFallbackAllowed: false, + }); + vi.mocked(SessionManager.releaseSessionDiscoveryLease).mockResolvedValue({ + status: "released", + legacyFallbackAllowed: false, + }); vi.mocked(SessionManager.updateSessionUsage).mockResolvedValue(undefined); vi.mocked(SessionManager.updateSessionBindingSmart).mockResolvedValue({ updated: true, @@ -383,6 +493,7 @@ function setupCommonMocks() { status: "settled", settlements: [], }); + vi.mocked(RateLimitService.releaseProviderSession).mockResolvedValue(undefined); vi.mocked(SessionTracker.refreshSession).mockResolvedValue(undefined); mockRecordFailure.mockResolvedValue(undefined); mockRecordSuccess.mockResolvedValue(undefined); @@ -428,6 +539,22 @@ describe("Endpoint circuit breaker isolation", () => { ).toBe(true); }); + it("does not clear a binding when the request routing mode forbids binding mutations", async () => { + const session = createSession(); + Object.assign(session, { isSessionBindingAllowed: () => false }); + setDeferredMeta(session, 42); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createFake200StreamResponse() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.clearSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.clearVersionedSessionProvider).not.toHaveBeenCalled(); + }); + it("高并发模式下，fake-200 流式错误仍应记录核心失败，但跳过 session 观测写入", async () => { const session = createSession(); session.setHighConcurrencyModeEnabled(true); @@ -537,4 +664,638 @@ describe("Endpoint circuit breaker isolation", () => { expect(mockRecordEndpointSuccess).not.toHaveBeenCalled(); expect(mockRecordEndpointFailure).not.toHaveBeenCalled(); }); + + it("does not clear a create binding when Discovery finishes with fake-200", async () => { + const session = createSession(); + session.recordProviderSessionRef(1); + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "create-generation", + }, + providerSessionRefOwned: true, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createFake200StreamResponse() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.clearVersionedSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.clearSessionProvider).not.toHaveBeenCalled(); + expect(RateLimitService.releaseProviderSession).toHaveBeenCalledOnce(); + expect(RateLimitService.releaseProviderSession).toHaveBeenCalledWith(1, "fake-session"); + }); + + it("clears only the captured renew snapshot after a fake-200", async () => { + const session = createSession(); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: 1, + generation: "renew-generation", + } as const; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 1, + isFirstAttempt: true, + isFailoverSuccess: false, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "renew", + bindingSnapshot: snapshot, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createFake200StreamResponse() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.clearVersionedSessionProvider).toHaveBeenCalledWith(snapshot, 1, 0); + expect(SessionManager.clearSessionProvider).not.toHaveBeenCalled(); + }); + + it("never mutates a binding for fallback intent none", async () => { + const session = createSession(); + session.recordProviderSessionRef(1); + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 2, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: false, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "none", + providerSessionRefOwned: true, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createFake200StreamResponse() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.clearVersionedSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.clearSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.updateSessionBindingSmart).not.toHaveBeenCalled(); + expect(RateLimitService.releaseProviderSession).toHaveBeenCalledWith(1, "fake-session"); + }); + + it("does not clear a create tombstone when the completion marker is missing", async () => { + const session = createSession(); + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "incomplete-generation", + }, + requiresCompletionMarker: true, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createSuccessStreamResponse() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.clearVersionedSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.clearSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); + }); + + it("does not accept completion marker words embedded in ordinary SSE content", async () => { + const session = createSession(); + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "misleading-content-generation", + }, + requiresCompletionMarker: true, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createMisleadingCompletionTextResponse() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); + expect(updateMessageRequestDetailsDurably).toHaveBeenCalledWith( + 1, + expect.objectContaining({ + statusCode: 502, + errorMessage: "STREAM_COMPLETION_MARKER_MISSING", + }), + expect.objectContaining({ onCommitted: expect.any(Function) }) + ); + }); + + it.each([ + { + label: "OpenAI Responses", + format: "response" as const, + body: `event: response.completed\ndata: ${JSON.stringify({ + type: "response.completed", + response: { id: "resp_completed" }, + })}\n\n`, + }, + { + label: "OpenAI Chat finish reason", + format: "openai" as const, + body: `data: ${JSON.stringify({ + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + })}\n\n`, + }, + { + label: "OpenAI Chat done sentinel", + format: "openai" as const, + body: "data: [DONE]\n\n", + }, + { + label: "Gemini", + format: "gemini" as const, + body: `data: ${JSON.stringify({ + candidates: [{ finishReason: "STOP" }], + })}\n\n`, + }, + { + label: "Gemini CLI", + format: "gemini-cli" as const, + body: `data: ${JSON.stringify({ + response: { candidates: [{ finishReason: "STOP" }] }, + })}\n\n`, + }, + ])("accepts a structurally valid $label completion marker", async ({ format, body }) => { + const session = createSession(); + session.originalFormat = format; + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: `${format}-completion-generation`, + } as const; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: snapshot, + requiresCompletionMarker: true, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }) + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.compareAndSetSessionProvider).toHaveBeenCalledWith(snapshot, 1); + }); + + it("releases a create attempt ref when generation CAS loses", async () => { + const session = createSession(); + session.recordProviderSessionRef(1); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "stale-generation", + } as const; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: snapshot, + requiresCompletionMarker: true, + providerSessionRefOwned: true, + providerSessionRefRetainOnSuccess: true, + }); + vi.mocked(SessionManager.compareAndSetSessionProvider).mockResolvedValueOnce({ + status: "conflict", + reason: "generation_mismatch", + legacyFallbackAllowed: false, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createSuccessStreamResponseWithCompletion() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(RateLimitService.releaseProviderSession).toHaveBeenCalledOnce(); + expect(RateLimitService.releaseProviderSession).toHaveBeenCalledWith(1, "fake-session"); + }); + + it("retains an owned Provider ref after a renew generation CAS succeeds", async () => { + const session = createSession(); + session.recordProviderSessionRef(1); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: 1, + generation: "renew-generation", + } as const; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 1, + isFirstAttempt: true, + isFailoverSuccess: false, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "renew", + bindingSnapshot: snapshot, + requiresCompletionMarker: true, + providerSessionRefOwned: true, + providerSessionRefRetainOnSuccess: true, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createSuccessStreamResponseWithCompletion() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.compareAndSetSessionProvider).toHaveBeenCalledWith(snapshot, 1); + expect(RateLimitService.releaseProviderSession).not.toHaveBeenCalled(); + }); + + it("releases an owned Provider ref after CAS success when it is not the new baseline", async () => { + const session = createSession(); + session.recordProviderSessionRef(1); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: 1, + generation: "existing-baseline-generation", + } as const; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 1, + isFirstAttempt: true, + isFailoverSuccess: false, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "renew", + bindingSnapshot: snapshot, + requiresCompletionMarker: true, + providerSessionRefOwned: true, + providerSessionRefRetainOnSuccess: false, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createSuccessStreamResponseWithCompletion() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.compareAndSetSessionProvider).toHaveBeenCalledWith(snapshot, 1); + expect(RateLimitService.releaseProviderSession).toHaveBeenCalledWith(1, "fake-session"); + }); + + it.each([ + { + label: "lost", + leaseResult: { + status: "lost", + reason: "not_owner_or_missing", + legacyFallbackAllowed: false, + } as const, + }, + { + label: "unavailable", + leaseResult: { + status: "unavailable", + reason: "operation_failed", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + } as const, + }, + ])("fails binding closed when the finalizer lease is $label", async ({ leaseResult }) => { + const session = createSession(); + session.recordProviderSessionRef(1); + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "lease-guarded-generation", + }, + requiresCompletionMarker: true, + discoveryLease: { + sessionId: "fake-session", + keyId: 456, + ownerToken: "lease-owner", + ttlSeconds: 30, + }, + providerSessionRefOwned: true, + }); + vi.mocked(SessionManager.renewSessionDiscoveryLease).mockResolvedValueOnce(leaseResult); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createSuccessStreamResponseWithCompletion() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.renewSessionDiscoveryLease).toHaveBeenCalledOnce(); + expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); + expect(RateLimitService.releaseProviderSession).toHaveBeenCalledWith(1, "fake-session"); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); + }); + + it("renews a long-stream lease and releases it once after terminal side effects", async () => { + vi.useFakeTimers(); + try { + const order: string[] = []; + const session = createSession(); + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 2, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: false, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "none", + requiresCompletionMarker: false, + discoveryLease: { + sessionId: "fake-session", + keyId: 456, + ownerToken: "lease-owner", + ttlSeconds: 2, + }, + }); + mockRecordSuccess.mockImplementationOnce(async () => { + order.push("side-effect"); + }); + vi.mocked(SessionManager.releaseSessionDiscoveryLease).mockImplementationOnce(async () => { + order.push("lease-release"); + return { status: "released", legacyFallbackAllowed: false }; + }); + const controlled = createControllableSuccessStreamResponse(); + + const clientResponse = await ProxyResponseHandler.dispatch(session, controlled.response); + const bodyPromise = clientResponse.text(); + await vi.advanceTimersByTimeAsync(0); + expect(SessionManager.renewSessionDiscoveryLease).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1_100); + expect(SessionManager.renewSessionDiscoveryLease).toHaveBeenCalledWith( + "fake-session", + 456, + "lease-owner", + 2 + ); + + controlled.complete(); + await bodyPromise; + await drainAsyncTasks(); + + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledWith( + "fake-session", + 456, + "lease-owner" + ); + expect(order).toEqual(["side-effect", "lease-release"]); + + const renewCalls = vi.mocked(SessionManager.renewSessionDiscoveryLease).mock.calls.length; + await vi.advanceTimersByTimeAsync(5_000); + expect(SessionManager.renewSessionDiscoveryLease).toHaveBeenCalledTimes(renewCalls); + } finally { + vi.useRealTimers(); + } + }); + + it("does not delay downstream delivery while the lease handoff renewal is pending", async () => { + const handoffRenewal = Promise.withResolvers<{ + status: "renewed"; + legacyFallbackAllowed: false; + }>(); + vi.mocked(SessionManager.renewSessionDiscoveryLease).mockReturnValueOnce( + handoffRenewal.promise + ); + const session = createSession(); + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 2, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: false, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "none", + requiresCompletionMarker: false, + discoveryLease: { + sessionId: "fake-session", + keyId: 456, + ownerToken: "pending-handoff-owner", + ttlSeconds: 30, + }, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createSuccessStreamResponseWithCompletion() + ); + await expect(clientResponse.text()).resolves.toContain("message_stop"); + expect(SessionManager.renewSessionDiscoveryLease).toHaveBeenCalledOnce(); + + handoffRenewal.resolve({ status: "renewed", legacyFallbackAllowed: false }); + await drainAsyncTasks(); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); + }); + + it("bounds a stalled lease release and still invokes compare-delete exactly once", async () => { + vi.useFakeTimers(); + try { + const session = createSession(); + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 2, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: false, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "none", + requiresCompletionMarker: false, + discoveryLease: { + sessionId: "fake-session", + keyId: 456, + ownerToken: "stalled-release-owner", + ttlSeconds: 30, + }, + }); + vi.mocked(SessionManager.releaseSessionDiscoveryLease).mockImplementationOnce( + () => new Promise(() => undefined) + ); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createSuccessStreamResponseWithCompletion() + ); + await clientResponse.text(); + const drainPromise = drainAsyncTasks(); + await vi.advanceTimersByTimeAsync(5_000); + await drainPromise; + + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledWith( + "fake-session", + 456, + "stalled-release-owner" + ); + } finally { + vi.useRealTimers(); + } + }); + + it("fails closed when the captured Discovery generation has expired", async () => { + const session = createSession(); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "expired-generation", + } as const; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: snapshot, + requiresCompletionMarker: true, + providerSessionRefOwned: true, + }); + session.recordProviderSessionRef(1); + vi.mocked(SessionManager.compareAndSetSessionProvider).mockResolvedValueOnce({ + status: "conflict", + reason: "canonical_missing", + legacyFallbackAllowed: false, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createSuccessStreamResponseWithCompletion() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.getSessionBindingSnapshot).not.toHaveBeenCalled(); + expect(SessionManager.compareAndSetSessionProvider).toHaveBeenCalledOnce(); + expect(SessionManager.compareAndSetSessionProvider).toHaveBeenCalledWith(snapshot, 1); + expect(RateLimitService.releaseProviderSession).toHaveBeenCalledWith(1, "fake-session"); + }); }); diff --git a/tests/unit/proxy/response-handler-gemini-stream-passthrough-timeouts.test.ts b/tests/unit/proxy/response-handler-gemini-stream-passthrough-timeouts.test.ts index ba7bd9b13..2524cc5ce 100644 --- a/tests/unit/proxy/response-handler-gemini-stream-passthrough-timeouts.test.ts +++ b/tests/unit/proxy/response-handler-gemini-stream-passthrough-timeouts.test.ts @@ -707,7 +707,7 @@ describe("ProxyResponseHandler - Gemini stream passthrough timeouts", () => { setTimeout(() => { try { res.end( - 'data: {"usageMetadata":{"promptTokenCount":3,"candidatesTokenCount":2},"finishReason":"STOP"}\n\n' + 'data: {"usageMetadata":{"promptTokenCount":3,"candidatesTokenCount":2},"candidates":[{"finishReason":"STOP"}]}\n\n' ); } catch { // ignore From 8ccaac3ce8609e9929720421b81d14a156f9c39c Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 19:35:43 -0400 Subject: [PATCH 42/89] fix(binding): preserve failover session metadata --- src/lib/redis/session-binding.ts | 9 ++++- src/lib/session-manager.ts | 39 +++++++++++++++++++ tests/unit/lib/redis/session-binding.test.ts | 7 +++- .../session-manager-terminate-session.test.ts | 35 +++++++++++++++++ 4 files changed, 88 insertions(+), 2 deletions(-) diff --git a/src/lib/redis/session-binding.ts b/src/lib/redis/session-binding.ts index 41713d0f1..8f0af980f 100644 --- a/src/lib/redis/session-binding.ts +++ b/src/lib/redis/session-binding.ts @@ -118,6 +118,8 @@ export type LegacySessionBindingMutationResult = status: "ok"; changed: boolean; providerId: number | null; + /** Provider removed at the scoped terminate linearization point. */ + terminatedProviderId?: number; } | SessionBindingConflictResult | SessionBindingUnavailableResult; @@ -1372,7 +1374,12 @@ export async function mutateLegacySessionBindingSafely( { value: providerToTerminate.toString(), ttlSeconds } ); if (conflictAfterTerminate) return conflictAfterTerminate; - return { status: "ok", changed: true, providerId: null }; + return { + status: "ok", + changed: true, + providerId: null, + terminatedProviderId: providerToTerminate, + }; } if (legacyProvider !== null) await redis.del(keys.legacyProvider); diff --git a/src/lib/session-manager.ts b/src/lib/session-manager.ts index d57a3a38e..58e56f144 100644 --- a/src/lib/session-manager.ts +++ b/src/lib/session-manager.ts @@ -2874,6 +2874,45 @@ export class SessionManager { }); return false; } + + if (expectedProviderIds) { + const terminatedProviderId = legacy.terminatedProviderId; + if (terminatedProviderId == null) { + logger.warn("SessionManager: Scoped legacy termination lost Provider identity", { + sessionId, + keyId, + }); + return false; + } + + // The helper's value-checked delete is the linearization point. A + // failover may bind this Session to Q immediately afterwards, so + // provider-scoped invalidation must not delete shared Session + // metadata or global/key/user indexes after removing P. + try { + const providerCleanup = redis.pipeline(); + providerCleanup.zrem(`provider:${terminatedProviderId}:active_sessions`, sessionId); + providerCleanup.hdel( + `provider:${terminatedProviderId}:active_session_refs`, + sessionId + ); + await providerCleanup.exec(); + } catch (cleanupError) { + logger.warn("SessionManager: Scoped legacy Provider index cleanup failed", { + sessionId, + providerId: terminatedProviderId, + error: cleanupError, + }); + } + + logger.info("SessionManager: Cleared scoped legacy Provider binding", { + sessionId, + providerId: terminatedProviderId, + keyId, + }); + return true; + } + bindingTerminated = true; } else { logger.warn("SessionManager: Session binding termination blocked", { diff --git a/tests/unit/lib/redis/session-binding.test.ts b/tests/unit/lib/redis/session-binding.test.ts index 7e235ab17..4a72e14d4 100644 --- a/tests/unit/lib/redis/session-binding.test.ts +++ b/tests/unit/lib/redis/session-binding.test.ts @@ -1407,7 +1407,12 @@ describe("versioned session binding operations", () => { redis: mock.redis, mutation: { type: "terminate", expectedProviderIds: [10] }, }) - ).resolves.toMatchObject({ status: "ok", changed: true, providerId: null }); + ).resolves.toMatchObject({ + status: "ok", + changed: true, + providerId: null, + terminatedProviderId: 10, + }); expect(owner).toBe("4"); }); diff --git a/tests/unit/lib/session-manager-terminate-session.test.ts b/tests/unit/lib/session-manager-terminate-session.test.ts index 7467b1f5d..17be57336 100644 --- a/tests/unit/lib/session-manager-terminate-session.test.ts +++ b/tests/unit/lib/session-manager-terminate-session.test.ts @@ -209,4 +209,39 @@ describe("SessionManager.terminateSession", () => { expect(pipelineRef.exec).not.toHaveBeenCalled(); expect(pipelineRef.del).not.toHaveBeenCalled(); }); + + it("preserves shared Session state after scoped legacy termination linearizes on the old Provider", async () => { + const sessionId = "sess_scoped_legacy_failover"; + redisClientRef.get.mockImplementation(async (key: string) => { + if (key === `session:${sessionId}:provider`) return "42"; + if (key === `session:${sessionId}:key`) return "7"; + return null; + }); + redisClientRef.hget.mockResolvedValue("123"); + bindingMocks.mutateLegacySessionBindingSafely.mockResolvedValueOnce({ + status: "ok", + changed: true, + providerId: null, + terminatedProviderId: 42, + }); + const { getGlobalActiveSessionsKey, getKeyActiveSessionsKey, getUserActiveSessionsKey } = + await import("@/lib/redis/active-session-keys"); + const { SessionManager } = await import("@/lib/session-manager"); + + await expect(SessionManager.terminateSession(sessionId, [42])).resolves.toBe(true); + + expect(bindingMocks.mutateLegacySessionBindingSafely).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId, + keyId: 7, + mutation: { type: "terminate", expectedProviderIds: [42] }, + }) + ); + expect(pipelineRef.zrem).toHaveBeenCalledWith("provider:42:active_sessions", sessionId); + expect(pipelineRef.hdel).toHaveBeenCalledWith("provider:42:active_session_refs", sessionId); + expect(pipelineRef.del).not.toHaveBeenCalled(); + expect(pipelineRef.zrem).not.toHaveBeenCalledWith(getGlobalActiveSessionsKey(), sessionId); + expect(pipelineRef.zrem).not.toHaveBeenCalledWith(getKeyActiveSessionsKey(7), sessionId); + expect(pipelineRef.zrem).not.toHaveBeenCalledWith(getUserActiveSessionsKey(123), sessionId); + }); }); From 0537a6e8574d03c774adea1d7c3aee34902f9daf Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 19:35:52 -0400 Subject: [PATCH 43/89] fix(discovery): retain blocked fallback readiness --- .../v1/_lib/proxy/discovery-coordinator.ts | 14 ++++ src/app/v1/_lib/proxy/forwarder.ts | 6 +- .../unit/proxy/discovery-coordinator.test.ts | 33 ++++++++ .../proxy-forwarder-hedge-first-byte.test.ts | 80 +++++++++++++++++++ 4 files changed, 131 insertions(+), 2 deletions(-) diff --git a/src/app/v1/_lib/proxy/discovery-coordinator.ts b/src/app/v1/_lib/proxy/discovery-coordinator.ts index f1330df92..b7879a8dc 100644 --- a/src/app/v1/_lib/proxy/discovery-coordinator.ts +++ b/src/app/v1/_lib/proxy/discovery-coordinator.ts @@ -150,6 +150,20 @@ export class DiscoveryCoordinator { return this.chooseReadyNormal(); } + /** Record a ready fallback without allowing it to preempt a reserved normal wave. */ + recordReadyHeld( + id: string, + requestEpoch = this.requestEpoch, + roundEpoch = this.roundEpoch + ): boolean { + if (!this.acceptsEpoch(requestEpoch, roundEpoch) || this.isTerminal) return false; + const attempt = this.attempts.get(id); + if (!attempt?.pending || attempt.kind !== "fallback") return false; + attempt.ready = true; + this.state = "FALLBACK_READY_HELD"; + return true; + } + /** Convert a timed-out Sticky attempt into the request's fallback lane. */ demoteToFallback( id: string, diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index e31a81b17..c9943e85d 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -5574,8 +5574,10 @@ export class ProxyForwarder { (fallbackPromotionBlocked || roundLaunchesInProgress > 0) ) { // The next wave has been reserved but its normal attempts may - // still be awaiting selection/endpoint setup. Keep the fallback - // ready-held until those slots are registered or exhausted. + // still be awaiting selection/endpoint setup. Persist readiness + // without promoting so the total deadline can still recover the + // buffered fallback if that setup stalls. + coordinator.recordReadyHeld(id); return; } // Record readiness even when the priority gate holds this attempt. diff --git a/tests/unit/proxy/discovery-coordinator.test.ts b/tests/unit/proxy/discovery-coordinator.test.ts index bdbb7ff20..1ce247f52 100644 --- a/tests/unit/proxy/discovery-coordinator.test.ts +++ b/tests/unit/proxy/discovery-coordinator.test.ts @@ -134,6 +134,39 @@ describe("DiscoveryCoordinator", () => { expect(coordinator.markFailed("fallback")).toEqual({ type: "none" }); }); + it("records a ready-held fallback without promoting it before the deadline", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("fallback", 1, "fallback")); + coordinator.addAttempt(attempt("normal", 1)); + + expect(coordinator.recordReadyHeld("fallback")).toBe(true); + expect(coordinator.snapshot.find((item) => item.id === "fallback")).toMatchObject({ + kind: "fallback", + ready: true, + pending: true, + }); + expect(coordinator.state).toBe("FALLBACK_READY_HELD"); + expect(coordinator.onDeadline()).toEqual({ + type: "promote_fallback", + attemptId: "fallback", + }); + }); + + it("rejects ready-held writes for stale, non-fallback, or inactive attempts", () => { + const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); + coordinator.addAttempt(attempt("normal", 1)); + coordinator.addAttempt(attempt("fallback", 1, "fallback")); + const staleEpoch = coordinator.epochs; + + expect(coordinator.recordReadyHeld("normal")).toBe(false); + coordinator.beginRound(); + expect( + coordinator.recordReadyHeld("fallback", staleEpoch.requestEpoch, staleEpoch.roundEpoch) + ).toBe(false); + coordinator.markFailed("fallback"); + expect(coordinator.recordReadyHeld("fallback")).toBe(false); + }); + it("keeps coordinator kind in sync when a running Sticky becomes fallback", () => { const coordinator = new DiscoveryCoordinator({ concurrency: 2, maxRounds: 2 }); coordinator.addAttempt(attempt("sticky", 1)); diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index bd2cafab4..f732a52ac 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -2509,6 +2509,86 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { } }); + test("a ready-held Sticky fallback survives a stalled next-wave selector until the total deadline", async () => { + vi.useFakeTimers(); + try { + const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 27 }, + apiKey: null, + } as typeof session.authState; + session.request.message.messages = [ + { role: "user", content: "first" }, + { role: "user", content: "second" }, + ]; + session.setProvider(sticky); + session.setSessionBindingSnapshot({ + sessionId: session.sessionId!, + keyId: 27, + providerId: sticky.id, + generation: "g-sticky-deadline", + }); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 20, + stickySlaMs: 10, + racingTotalTimeoutMs: 50, + stickyTimeoutCooldownMs: 300_000, + }); + + const stalledSelector = Promise.withResolvers(); + mocks.pickDiscoveryProviders.mockReturnValueOnce(stalledSelector.promise); + vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ).mockResolvedValueOnce( + new Response( + new ReadableStream({ + start(controller) { + setTimeout(() => { + controller.enqueue( + new TextEncoder().encode( + 'data: {"type":"content_block_delta","delta":{"text":"sticky-fallback"}}\n\n' + ) + ); + controller.close(); + }, 15); + }, + }), + { headers: { "content-type": "text/event-stream" } } + ) + ); + + let settledEarly = false; + const responsePromise = ProxyForwarder.send(session).then((response) => { + settledEarly = true; + return response; + }); + + await vi.advanceTimersByTimeAsync(10); + expect(mocks.pickDiscoveryProviders).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(5); + expect(settledEarly).toBe(false); + + await vi.advanceTimersByTimeAsync(35); + const response = await responsePromise; + expect(await response.text()).toContain('"sticky-fallback"'); + expect(session.provider?.id).toBe(sticky.id); + + stalledSelector.resolve([]); + await vi.advanceTimersByTimeAsync(0); + } finally { + vi.useRealTimers(); + } + }); + test("Sticky probing does not consume a configured Discovery round", async () => { vi.useFakeTimers(); try { From c433d35df01206d7f10cae975041dd6c3ab2a6e5 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 19:58:05 -0400 Subject: [PATCH 44/89] fix(binding): preserve scoped versioned session state --- src/lib/session-manager.ts | 38 ++++++++++++++ .../session-manager-terminate-session.test.ts | 51 +++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/src/lib/session-manager.ts b/src/lib/session-manager.ts index 58e56f144..16e54c285 100644 --- a/src/lib/session-manager.ts +++ b/src/lib/session-manager.ts @@ -2857,6 +2857,44 @@ export class SessionManager { }); return false; } + + if (expectedProviderIds) { + const terminatedProviderId = binding.snapshot.providerId; + if (terminatedProviderId === null) { + logger.warn("SessionManager: Scoped versioned termination lost Provider identity", { + sessionId, + keyId, + }); + return false; + } + + // The versioned CAS above is the linearization point. A failover + // may bind this Session to Q immediately afterwards, so scoped + // invalidation must only remove P's Provider-owned indexes. + try { + const providerCleanup = redis.pipeline(); + providerCleanup.zrem(`provider:${terminatedProviderId}:active_sessions`, sessionId); + providerCleanup.hdel( + `provider:${terminatedProviderId}:active_session_refs`, + sessionId + ); + await providerCleanup.exec(); + } catch (cleanupError) { + logger.warn("SessionManager: Scoped versioned Provider index cleanup failed", { + sessionId, + providerId: terminatedProviderId, + error: cleanupError, + }); + } + + logger.info("SessionManager: Cleared scoped versioned Provider binding", { + sessionId, + providerId: terminatedProviderId, + keyId, + }); + return true; + } + bindingTerminated = true; } else if (binding.status === "unavailable" && binding.legacyFallbackAllowed) { const legacy = await mutateLegacySessionBindingSafely({ diff --git a/tests/unit/lib/session-manager-terminate-session.test.ts b/tests/unit/lib/session-manager-terminate-session.test.ts index 17be57336..05f6ac1b2 100644 --- a/tests/unit/lib/session-manager-terminate-session.test.ts +++ b/tests/unit/lib/session-manager-terminate-session.test.ts @@ -180,6 +180,57 @@ describe("SessionManager.terminateSession", () => { expect(pipelineRef.del).not.toHaveBeenCalledWith(`session:${sessionId}:key`); }); + it("preserves shared Session state after scoped versioned termination linearizes on the old Provider", async () => { + const sessionId = "sess_scoped_versioned_failover"; + redisClientRef.get.mockImplementation(async (key: string) => { + if (key === `session:${sessionId}:provider`) return "42"; + if (key === `session:${sessionId}:key`) return "7"; + return null; + }); + redisClientRef.hget.mockResolvedValue("123"); + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "ok", + source: "existing", + snapshot: { + sessionId, + keyId: 7, + providerId: 42, + generation: "generation-before-provider-invalidation", + }, + legacyFallbackAllowed: false, + }); + bindingMocks.terminateSessionBinding.mockResolvedValue({ + status: "ok", + source: "terminated", + snapshot: { + sessionId, + keyId: 7, + providerId: null, + generation: "generation-after-provider-invalidation", + }, + legacyFallbackAllowed: false, + }); + const { getGlobalActiveSessionsKey, getKeyActiveSessionsKey, getUserActiveSessionsKey } = + await import("@/lib/redis/active-session-keys"); + const { SessionManager } = await import("@/lib/session-manager"); + + await expect(SessionManager.terminateSession(sessionId, [42])).resolves.toBe(true); + + expect(bindingMocks.terminateSessionBinding).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId, + keyId: 7, + expectedProviderId: 42, + }) + ); + expect(pipelineRef.zrem).toHaveBeenCalledWith("provider:42:active_sessions", sessionId); + expect(pipelineRef.hdel).toHaveBeenCalledWith("provider:42:active_session_refs", sessionId); + expect(pipelineRef.del).not.toHaveBeenCalled(); + expect(pipelineRef.zrem).not.toHaveBeenCalledWith(getGlobalActiveSessionsKey(), sessionId); + expect(pipelineRef.zrem).not.toHaveBeenCalledWith(getKeyActiveSessionsKey(7), sessionId); + expect(pipelineRef.zrem).not.toHaveBeenCalledWith(getUserActiveSessionsKey(123), sessionId); + }); + it("does not delete session metadata when versioned termination hits a mirror conflict", async () => { const sessionId = "sess_mirror_conflict"; redisClientRef.get.mockImplementation(async (key: string) => { From 12887e497070adbb4fcf088c8ccdfe9fe525b08d Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 20:00:33 -0400 Subject: [PATCH 45/89] fix(discovery): reserve sticky timeout wave --- src/app/v1/_lib/proxy/forwarder.ts | 24 +++- .../proxy-forwarder-hedge-first-byte.test.ts | 110 ++++++++++++++++++ 2 files changed, 130 insertions(+), 4 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index c9943e85d..1f1489c00 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -5041,6 +5041,7 @@ export class ProxyForwarder { let roundLaunchesInProgress = 0; const roundLaunchIdleWaiters = new Set<() => void>(); let fallbackPromotionBlocked = false; + let stickyTimeoutWaveReservation: { fallbackAttemptId: string } | null = null; const hasSticky = session.shouldReuseProvider() && !!session.sessionId && @@ -5781,7 +5782,14 @@ export class ProxyForwarder { failureAction.type === "launch" || failureAction.type === "terminal_failure"; if (actionOwnsNextStep) { - await executeCoordinatorAction(failureAction); + if ( + failureAction.type === "launch" && + stickyTimeoutWaveReservation?.fallbackAttemptId === id + ) { + await launchReservedStickyTimeoutWave(failureAction.slots); + } else { + await executeCoordinatorAction(failureAction); + } } if (!actionOwnsNextStep && !committed && !settled) { const replacement = await chooseCandidate(); @@ -5868,6 +5876,13 @@ export class ProxyForwarder { } }; + const launchReservedStickyTimeoutWave = async (slots: number): Promise => { + if (!stickyTimeoutWaveReservation) return; + stickyTimeoutWaveReservation = null; + if (settled || committed) return; + await launchNextRound(slots, true); + }; + executeCoordinatorAction = async (action, terminalCancellationKind) => { if (settled || committed) return; if (action.type === "cancel" || action.type === "launch") { @@ -5965,17 +5980,18 @@ export class ProxyForwarder { sticky.kind = "fallback"; discoveryMetrics.fallbackPromoted(sticky.id, sticky.provider.id, 0); fallbackPromotionBlocked = true; + stickyTimeoutWaveReservation = { fallbackAttemptId: sticky.id }; if (bindingSnapshot && bindingSnapshot.providerId === initialProvider.id) { void clearCapturedStickyBinding( Math.ceil((settings.stickyTimeoutCooldownMs ?? 300_000) / 1000) ).finally(() => { - void launchNextRound(Math.max(0, concurrency - 1), true).catch((error) => - logger.warn("[Discovery] Sticky round launch failed", { error }) + void launchReservedStickyTimeoutWave(Math.max(0, concurrency - 1)).catch( + (error) => logger.warn("[Discovery] Sticky round launch failed", { error }) ); }); return; } - void launchNextRound(Math.max(0, concurrency - 1), true).catch((error) => + void launchReservedStickyTimeoutWave(Math.max(0, concurrency - 1)).catch((error) => logger.warn("[Discovery] Sticky round launch failed", { error }) ); } diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index f732a52ac..7bbcc1856 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -2509,6 +2509,116 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { } }); + test("Sticky timeout and fallback failure consume a single replacement-wave reservation", async () => { + vi.useFakeTimers(); + try { + const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); + const normalOne = createProvider({ id: 2, name: "normal-one", priority: 1 }); + const normalTwo = createProvider({ id: 3, name: "normal-two", priority: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 28 }, + apiKey: null, + } as typeof session.authState; + session.request.message.messages = [ + { role: "user", content: "first" }, + { role: "user", content: "second" }, + ]; + session.setProvider(sticky); + session.setSessionBindingSnapshot({ + sessionId: session.sessionId!, + keyId: 28, + providerId: sticky.id, + generation: "g-sticky-single-wave", + }); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 2, + discoverySlaMs: 50, + stickySlaMs: 10, + racingTotalTimeoutMs: 200, + stickyTimeoutCooldownMs: 300_000, + }); + + const clearBinding = Promise.withResolvers(); + mocks.clearVersionedSessionProvider.mockReturnValueOnce(clearBinding.promise); + const replacementWave = Promise.withResolvers(); + mocks.pickDiscoveryProviders.mockReturnValueOnce(replacementWave.promise); + + const stickyAttempt = Promise.withResolvers(); + const normalOneAttempt = Promise.withResolvers(); + const normalTwoAttempt = Promise.withResolvers(); + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockImplementation(async (attemptSession) => { + switch ((attemptSession as ProxySession).provider?.id) { + case sticky.id: + return stickyAttempt.promise; + case normalOne.id: + return normalOneAttempt.promise; + case normalTwo.id: + return normalTwoAttempt.promise; + default: + throw new Error("unexpected Provider"); + } + }); + + const responsePromise = ProxyForwarder.send(session); + await vi.advanceTimersByTimeAsync(10); + expect(mocks.clearVersionedSessionProvider).toHaveBeenCalledTimes(1); + expect(mocks.pickDiscoveryProviders).not.toHaveBeenCalled(); + + stickyAttempt.reject(new Error("Sticky fallback failed while binding clear was pending")); + await vi.advanceTimersByTimeAsync(0); + expect(mocks.pickDiscoveryProviders).toHaveBeenCalledTimes(1); + expect(mocks.pickDiscoveryProviders).toHaveBeenCalledWith( + expect.anything(), + 2, + expect.arrayContaining([sticky.id]) + ); + + clearBinding.resolve({ + status: "ok", + legacyFallbackAllowed: false, + source: "cleared", + snapshot: { + sessionId: session.sessionId!, + keyId: 28, + providerId: null, + generation: "g-cleared-single-wave", + }, + }); + await vi.advanceTimersByTimeAsync(0); + expect(mocks.pickDiscoveryProviders).toHaveBeenCalledTimes(1); + + replacementWave.resolve([normalOne, normalTwo]); + await vi.advanceTimersByTimeAsync(0); + expect(doForward).toHaveBeenCalledTimes(3); + + normalOneAttempt.resolve( + new Response('data: {"type":"content_block_delta","delta":{"text":"normal-one"}}\n\n', { + headers: { "content-type": "text/event-stream" }, + }) + ); + await vi.advanceTimersByTimeAsync(0); + const response = await responsePromise; + expect(await response.text()).toContain('"normal-one"'); + expect(session.provider?.id).toBe(normalOne.id); + + normalTwoAttempt.resolve(new Response(null)); + await vi.advanceTimersByTimeAsync(0); + } finally { + vi.useRealTimers(); + } + }); + test("a ready-held Sticky fallback survives a stalled next-wave selector until the total deadline", async () => { vi.useFakeTimers(); try { From 08337d8feac1df1e857ffa3fa096e9da218a6684 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 20:05:32 -0400 Subject: [PATCH 46/89] fix(discovery): stop parsing after validity errors --- src/app/v1/_lib/proxy/discovery-validity.ts | 9 ++++++++- tests/unit/proxy/discovery-validity.test.ts | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/app/v1/_lib/proxy/discovery-validity.ts b/src/app/v1/_lib/proxy/discovery-validity.ts index 3325295a7..059bde293 100644 --- a/src/app/v1/_lib/proxy/discovery-validity.ts +++ b/src/app/v1/_lib/proxy/discovery-validity.ts @@ -155,6 +155,7 @@ export class DiscoveryValidityParser { constructor(readonly protocol: DiscoveryProtocol) {} push(chunk: Uint8Array | string): DiscoveryValidity { + if (this._error) return this.result; this.bytesSeen += typeof chunk === "string" ? new TextEncoder().encode(chunk).byteLength : chunk.byteLength; if (!this._ready && this.bytesSeen > DISCOVERY_PREFIX_MAX_BYTES) { @@ -172,7 +173,13 @@ export class DiscoveryValidityParser { if (this.buffered.includes("\n")) { const lines = this.buffered.split(/\r?\n/); this.buffered = lines.pop() ?? ""; - for (const line of lines) this.consumeLine(line); + for (const line of lines) { + this.consumeLine(line); + if (this._error) { + this.buffered = ""; + return this.result; + } + } } // Some providers return one raw JSON object without an SSE newline. Parse diff --git a/tests/unit/proxy/discovery-validity.test.ts b/tests/unit/proxy/discovery-validity.test.ts index 9c5ca8b39..5093298dd 100644 --- a/tests/unit/proxy/discovery-validity.test.ts +++ b/tests/unit/proxy/discovery-validity.test.ts @@ -139,6 +139,24 @@ describe("discovery validity", () => { } expect(result).toMatchObject({ ready: false, error: true, limitExceeded: true }); }); + + it("stops parsing current and future events after the event limit", () => { + const parser = new DiscoveryValidityParser("anthropic"); + const metadataEvents = Array.from( + { length: DISCOVERY_EVENT_MAX_COUNT + 1 }, + () => 'data: {"type":"ping"}\n' + ).join(""); + + const limited = parser.push(`${metadataEvents}data: {"type":"message_stop"}\n`); + expect(limited).toMatchObject({ + ready: false, + terminal: false, + error: true, + limitExceeded: true, + }); + + expect(parser.push('data: {"type":"message_stop"}\n')).toEqual(limited); + }); }); function parserForOpenAIChatToolCall(): DiscoveryValidityParser { From 95a94e80c45ff265d5bc650c873bd54843a227aa Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 20:07:47 -0400 Subject: [PATCH 47/89] feat(binding): add snapshot-safe TTL touch --- src/lib/redis/lua-scripts.ts | 84 +++++++++++++++++ src/lib/redis/session-binding.ts | 83 +++++++++++++++- src/lib/session-manager.ts | 24 +++++ .../session-binding-versioning-redis.test.ts | 90 ++++++++++++++++++ tests/unit/lib/redis/session-binding.test.ts | 94 +++++++++++++++++-- .../session-manager-versioned-binding.test.ts | 27 ++++++ 6 files changed, 394 insertions(+), 8 deletions(-) diff --git a/src/lib/redis/lua-scripts.ts b/src/lib/redis/lua-scripts.ts index 4e3437dad..6b4a9829b 100644 --- a/src/lib/redis/lua-scripts.ts +++ b/src/lib/redis/lua-scripts.ts @@ -592,6 +592,90 @@ redis.call('SETEX', legacy_provider_key, ttl, next_provider_id) return {'ok', 'updated', next_generation, next_provider_id} `; +/** + * Extend an existing binding's TTL only while the complete captured snapshot + * still matches canonical state and both legacy mirrors. This operation never + * initializes a missing binding or rotates its generation. + * + * KEYS[1]: canonical binding hash + * KEYS[2]: legacy provider string + * KEYS[3]: legacy key owner string + * ARGV[1]: current key id + * ARGV[2]: expected generation + * ARGV[3]: expected provider id, or empty for a null binding + * ARGV[4]: binding TTL in seconds + */ +export const TOUCH_SESSION_BINDING = ` +local binding_key = KEYS[1] +local legacy_provider_key = KEYS[2] +local legacy_owner_key = KEYS[3] + +local current_key_id = ARGV[1] +local expected_generation = ARGV[2] +local expected_provider_id = ARGV[3] +local ttl = tonumber(ARGV[4]) + +local function is_positive_integer(value) + local parsed = tonumber(value) + return parsed and parsed > 0 and parsed == math.floor(parsed) +end + +if not ttl or ttl <= 0 or current_key_id == '' or expected_generation == '' or + (expected_provider_id ~= '' and not is_positive_integer(expected_provider_id)) then + return {'conflict', 'invalid_input'} +end +if redis.call('EXISTS', binding_key) == 0 then + return {'conflict', 'canonical_missing'} +end + +local binding = redis.call('HMGET', binding_key, 'key_id', 'generation', 'provider_id') +local binding_key_id = binding[1] +local generation = binding[2] +local current_provider_id = binding[3] + +if not binding_key_id or not generation or binding_key_id == '' or generation == '' then + return {'conflict', 'canonical_corrupt'} +end +if binding_key_id ~= current_key_id then + return {'conflict', 'canonical_key_mismatch'} +end +if current_provider_id and not is_positive_integer(current_provider_id) then + return {'conflict', 'canonical_corrupt'} +end +if generation ~= expected_generation then + return {'conflict', 'generation_mismatch'} +end +if (current_provider_id or '') ~= expected_provider_id then + return {'conflict', 'provider_mismatch'} +end + +local legacy_owner = redis.call('GET', legacy_owner_key) +local legacy_provider = redis.call('GET', legacy_provider_key) +if legacy_provider and not is_positive_integer(legacy_provider) then + return {'conflict', 'invalid_legacy_provider'} +end +if not legacy_owner then + return {'conflict', 'mirror_missing'} +end +if legacy_owner ~= current_key_id then + return {'conflict', 'foreign_legacy_owner'} +end +if current_provider_id then + if legacy_provider ~= current_provider_id then + return {'conflict', 'mirror_conflict'} + end +elseif legacy_provider then + return {'conflict', 'mirror_conflict'} +end + +redis.call('EXPIRE', binding_key, ttl) +redis.call('EXPIRE', legacy_owner_key, ttl) +if current_provider_id then + redis.call('EXPIRE', legacy_provider_key, ttl) +end +return {'ok', 'touched', generation, current_provider_id or ''} +`; + /** * Compare-and-clear an existing versioned session binding. A cooldown marker * may be written in the same transaction when clearing a timed-out provider. diff --git a/src/lib/redis/session-binding.ts b/src/lib/redis/session-binding.ts index 8f0af980f..327979d9d 100644 --- a/src/lib/redis/session-binding.ts +++ b/src/lib/redis/session-binding.ts @@ -12,6 +12,7 @@ import { RENEW_SESSION_DISCOVERY_LEASE, RESTORE_LEGACY_PROVIDER_IF_ABSENT, TERMINATE_SESSION_BINDING, + TOUCH_SESSION_BINDING, } from "./lua-scripts"; export const DEFAULT_SESSION_BINDING_TTL_SECONDS = 300; @@ -93,6 +94,11 @@ export interface ClearSessionBindingInput extends ReadOrReconcileSessionBindingI cooldownTtlSeconds?: number; } +export interface TouchSessionBindingInput extends ReadOrReconcileSessionBindingInput { + expectedGeneration: string; + expectedProviderId: number | null; +} + export interface TerminateSessionBindingInput extends ReadOrReconcileSessionBindingInput { expectedProviderId?: number; } @@ -179,7 +185,14 @@ export interface SessionBindingOkResult { status: "ok"; snapshot: SessionBindingSnapshot; legacyFallbackAllowed: false; - source: "created" | "existing" | "legacy_upgraded" | "updated" | "cleared" | "terminated"; + source: + | "created" + | "existing" + | "legacy_upgraded" + | "updated" + | "touched" + | "cleared" + | "terminated"; } export interface SessionBindingConflictResult { @@ -225,6 +238,7 @@ interface ReadyClient { const READ_SOURCES = new Set(["created", "existing", "legacy_upgraded"]); const MUTATION_SOURCES = new Set(["updated", "cleared"]); +const TOUCH_SOURCES = new Set(["touched"]); const CONFLICT_REASONS = new Set([ "canonical_corrupt", "canonical_exists", @@ -609,6 +623,33 @@ async function runCapabilityProbe( throw new Error("Session binding capability probe update failed"); } + const touched = parseBindingResult( + await withCapabilityProbeDeadline( + client.eval( + TOUCH_SESSION_BINDING, + 3, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + keyId.toString(), + updated.snapshot.generation, + providerId.toString(), + ttlSeconds.toString() + ), + deadlineAt + ), + { sessionId, keyId }, + TOUCH_SOURCES + ); + if ( + touched.status !== "ok" || + touched.source !== "touched" || + touched.snapshot.generation !== updated.snapshot.generation || + touched.snapshot.providerId !== providerId + ) { + throw new Error("Session binding capability probe touch failed"); + } + const cleared = parseBindingResult( await withCapabilityProbeDeadline( client.eval( @@ -971,6 +1012,46 @@ export async function compareAndSetSessionBinding( } } +export async function touchSessionBinding( + input: TouchSessionBindingInput +): Promise { + const ttlSeconds = input.ttlSeconds ?? DEFAULT_SESSION_BINDING_TTL_SECONDS; + if ( + !isValidIdentity(input.sessionId, input.keyId, ttlSeconds) || + !input.expectedGeneration || + (input.expectedProviderId !== null && !isPositiveInteger(input.expectedProviderId)) + ) { + return conflict("invalid_input"); + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + const keys = buildSessionBindingKeys(input.sessionId, input.keyId); + try { + const raw = await evalBindingScript( + ready.redis, + TOUCH_SESSION_BINDING, + 3, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + input.keyId.toString(), + input.expectedGeneration, + input.expectedProviderId?.toString() ?? "", + ttlSeconds.toString() + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return parseBindingResult( + raw, + { sessionId: input.sessionId, keyId: input.keyId }, + TOUCH_SOURCES + ); + } catch (error) { + return handleOperationError(ready, error); + } +} + export async function clearSessionBinding( input: ClearSessionBindingInput ): Promise { diff --git a/src/lib/session-manager.ts b/src/lib/session-manager.ts index 16e54c285..341b407a3 100644 --- a/src/lib/session-manager.ts +++ b/src/lib/session-manager.ts @@ -49,6 +49,7 @@ import { type SessionDiscoveryLeaseMutationResult, type SessionProviderCooldownResult, terminateSessionBinding as terminateVersionedSessionBinding, + touchSessionBinding, type VersionedBindingCapabilityState, } from "./redis/session-binding"; import { SessionTracker } from "./session-tracker"; @@ -736,6 +737,29 @@ export class SessionManager { }); } + /** + * Heartbeats run at one third of the configured binding TTL, leaving time + * for a transient Redis failure without allowing a live binding to expire. + */ + static getVersionedSessionBindingRefreshIntervalMs(): number { + return Math.max(1, Math.floor((SessionManager.SESSION_TTL * 1000) / 3)); + } + + static async touchVersionedSessionBinding( + snapshot: SessionBindingSnapshot + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return touchSessionBinding({ + sessionId: snapshot.sessionId, + keyId: snapshot.keyId, + expectedGeneration: snapshot.generation, + expectedProviderId: snapshot.providerId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + } + static async compareAndSetSessionProvider( snapshot: SessionBindingSnapshot, providerId: number diff --git a/tests/integration/session-binding-versioning-redis.test.ts b/tests/integration/session-binding-versioning-redis.test.ts index 2520d9e98..3459f7dba 100644 --- a/tests/integration/session-binding-versioning-redis.test.ts +++ b/tests/integration/session-binding-versioning-redis.test.ts @@ -13,6 +13,8 @@ import { isSessionProviderCoolingDown, readOrReconcileSessionBinding, resetVersionedBindingCapabilityForTests, + terminateSessionBinding, + touchSessionBinding, type SessionBindingOkResult, type SessionBindingResult, } from "@/lib/redis/session-binding"; @@ -378,6 +380,94 @@ runWithVersionedBinding("versioned session binding reconcile", () => { }); runWithVersionedBinding("versioned session binding mutation", () => { + test("touches exact null and provider snapshots without rotating generation", async () => { + const sessionId = nextSessionId("touch-exact"); + const keyId = 1200; + const providerId = 2200; + const keys = buildSessionBindingKeys(sessionId, keyId); + rememberBindingKeys(sessionId, [keyId]); + + const initial = requireOk(await readBinding(sessionId, keyId)); + await redis.expire(keys.canonical, 5); + await redis.expire(keys.legacyOwner, 5); + const touchedNull = requireOk( + await touchSessionBinding({ + ...initial.snapshot, + expectedGeneration: initial.snapshot.generation, + expectedProviderId: null, + ttlSeconds: BINDING_TTL_SECONDS, + redis, + }) + ); + expect(touchedNull.source).toBe("touched"); + expect(touchedNull.snapshot).toEqual(initial.snapshot); + expect(await redis.ttl(keys.canonical)).toBeGreaterThan(BINDING_TTL_SECONDS - 5); + expect(await redis.ttl(keys.legacyOwner)).toBeGreaterThan(BINDING_TTL_SECONDS - 5); + expect(await redis.exists(keys.legacyProvider)).toBe(0); + + const bound = requireOk( + await bindProvider(sessionId, keyId, touchedNull.snapshot.generation, providerId) + ); + await redis.expire(keys.canonical, 5); + await redis.expire(keys.legacyOwner, 5); + await redis.expire(keys.legacyProvider, 5); + const touchedProvider = requireOk( + await touchSessionBinding({ + sessionId, + keyId, + expectedGeneration: bound.snapshot.generation, + expectedProviderId: providerId, + ttlSeconds: BINDING_TTL_SECONDS, + redis, + }) + ); + expect(touchedProvider.source).toBe("touched"); + expect(touchedProvider.snapshot).toEqual(bound.snapshot); + expect(await redis.ttl(keys.canonical)).toBeGreaterThan(BINDING_TTL_SECONDS - 5); + expect(await redis.ttl(keys.legacyOwner)).toBeGreaterThan(BINDING_TTL_SECONDS - 5); + expect(await redis.ttl(keys.legacyProvider)).toBeGreaterThan(BINDING_TTL_SECONDS - 5); + }); + + test("rejects a stale touch after administrative termination advances generation", async () => { + const sessionId = nextSessionId("touch-after-admin-termination"); + const keyId = 1201; + const providerId = 2201; + const keys = buildSessionBindingKeys(sessionId, keyId); + rememberBindingKeys(sessionId, [keyId]); + + const initial = requireOk(await readBinding(sessionId, keyId)); + const bound = requireOk( + await bindProvider(sessionId, keyId, initial.snapshot.generation, providerId) + ); + const terminated = requireOk( + await terminateSessionBinding({ + sessionId, + keyId, + expectedProviderId: providerId, + ttlSeconds: BINDING_TTL_SECONDS, + redis, + }) + ); + + const staleTouch = await touchSessionBinding({ + sessionId, + keyId, + expectedGeneration: bound.snapshot.generation, + expectedProviderId: providerId, + ttlSeconds: BINDING_TTL_SECONDS, + redis, + }); + + expect(staleTouch).toEqual({ + status: "conflict", + reason: "generation_mismatch", + legacyFallbackAllowed: false, + }); + expect(await redis.hget(keys.canonical, "generation")).toBe(terminated.snapshot.generation); + expect(await redis.hget(keys.canonical, "provider_id")).toBeNull(); + expect(await redis.exists(keys.legacyProvider)).toBe(0); + }); + test("rotates generation across CAS and rejects a stale ABA clear", async () => { const sessionId = nextSessionId("aba"); const keyId = 1201; diff --git a/tests/unit/lib/redis/session-binding.test.ts b/tests/unit/lib/redis/session-binding.test.ts index 4a72e14d4..a3132ada5 100644 --- a/tests/unit/lib/redis/session-binding.test.ts +++ b/tests/unit/lib/redis/session-binding.test.ts @@ -25,6 +25,7 @@ import { renewSessionDiscoveryLease, resetVersionedBindingCapabilityForTests, terminateSessionBinding, + touchSessionBinding, type SessionBindingRedisClient, } from "@/lib/redis/session-binding"; import { @@ -36,6 +37,7 @@ import { RENEW_SESSION_DISCOVERY_LEASE, RESTORE_LEGACY_PROVIDER_IF_ABSENT, TERMINATE_SESSION_BINDING, + TOUCH_SESSION_BINDING, } from "@/lib/redis/lua-scripts"; type EvalResponse = unknown | Error | ((args: unknown[]) => unknown | Promise); @@ -73,6 +75,9 @@ function createMockRedis(options: MockRedisOptions = {}) { if (script === CAS_SESSION_BINDING) { return ["ok", "updated", String(args[7]), String(args[8])]; } + if (script === TOUCH_SESSION_BINDING) { + return ["ok", "touched", String(args[6]), String(args[7])]; + } if (script === CLEAR_SESSION_BINDING) { probeCooldowns.set(String(args[5]), String(args[8])); return ["ok", "cleared", String(args[8]), ""]; @@ -210,14 +215,14 @@ describe("versioned binding capability", () => { resetVersionedBindingCapabilityForTests(); }); - it("probes reconcile, CAS, clear, cooldown, and cleanup exactly once per connection", async () => { + it("probes reconcile, CAS, touch, clear, cooldown, and cleanup exactly once per connection", async () => { const mock = createMockRedis(); await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("available"); await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("available"); expect(getVersionedBindingCapabilityState()).toBe("available"); - expect(mock.evalMock).toHaveBeenCalledTimes(5); + expect(mock.evalMock).toHaveBeenCalledTimes(6); expect(mock.getMock).toHaveBeenCalledTimes(1); expect(mock.setMock).toHaveBeenCalledTimes(1); expect(mock.delMock).toHaveBeenCalledTimes(5); @@ -244,7 +249,7 @@ describe("versioned binding capability", () => { releaseProbe?.(); await expect(Promise.all([first, second])).resolves.toEqual(["available", "available"]); - expect(mock.evalMock).toHaveBeenCalledTimes(5); + expect(mock.evalMock).toHaveBeenCalledTimes(6); }); it("stays unavailable on the same connection after a capability failure", async () => { @@ -267,7 +272,7 @@ describe("versioned binding capability", () => { expect(getVersionedBindingCapabilityState()).toBe("unknown"); await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("available"); - expect(mock.evalMock).toHaveBeenCalledTimes(6); + expect(mock.evalMock).toHaveBeenCalledTimes(7); }); it("automatically probes when a reconnected client becomes ready", async () => { @@ -280,14 +285,14 @@ describe("versioned binding capability", () => { mock.emit("ready"); await vi.waitFor(() => expect(getVersionedBindingCapabilityState()).toBe("available")); - expect(mock.evalMock).toHaveBeenCalledTimes(6); + expect(mock.evalMock).toHaveBeenCalledTimes(7); }); it("does not become available when isolated probe cleanup fails", async () => { const mock = createMockRedis({ cleanupFails: true }); await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("unavailable"); - expect(mock.evalMock).toHaveBeenCalledTimes(5); + expect(mock.evalMock).toHaveBeenCalledTimes(6); expect(mock.delMock).toHaveBeenCalledTimes(5); }); @@ -659,6 +664,81 @@ describe("versioned session binding operations", () => { expect(getVersionedBindingCapabilityState()).toBe("available"); }); + it.each([ + { label: "null tombstone", providerId: null }, + { label: "provider binding", providerId: 23 }, + ])("touches an exact $label without rotating generation", async ({ providerId }) => { + const mock = createMockRedis({ + operationResponses: { + [TOUCH_SESSION_BINDING]: [(args) => ["ok", "touched", String(args[6]), String(args[7])]], + }, + }); + + const result = await touchSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "captured-generation", + expectedProviderId: providerId, + ttlSeconds: 90, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "ok", + source: "touched", + snapshot: { + sessionId: "sid", + keyId: 4, + providerId, + generation: "captured-generation", + }, + legacyFallbackAllowed: false, + }); + const operation = mock.evalMock.mock.calls.at(-1); + expect(operation?.slice(0, 5)).toEqual([ + TOUCH_SESSION_BINDING, + 3, + buildCanonicalSessionBindingKey("sid", 4), + "session:sid:provider", + "session:sid:key", + ]); + expect(operation?.slice(5)).toEqual([ + "4", + "captured-generation", + providerId?.toString() ?? "", + "90", + ]); + }); + + it.each([ + ["an advanced generation", "generation_mismatch"], + ["a different provider", "provider_mismatch"], + ["a missing canonical binding", "canonical_missing"], + ["a missing legacy mirror", "mirror_missing"], + ["a contradictory legacy mirror", "mirror_conflict"], + ] as const)("fails closed when touching %s", async (_label, reason) => { + const mock = createMockRedis({ + operationResponses: { + [TOUCH_SESSION_BINDING]: [["conflict", reason]], + }, + }); + + const result = await touchSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "stale-or-conflicting-generation", + expectedProviderId: 23, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "conflict", + reason, + legacyFallbackAllowed: false, + }); + expect(getVersionedBindingCapabilityState()).toBe("available"); + }); + it("clears a provider with a tenant-scoped cooldown in the same Lua call", async () => { const mock = createMockRedis({ cooldownValue: "cooldown-generation", @@ -846,7 +926,7 @@ describe("versioned session binding operations", () => { keyId: 4, redis: mock.redis, }); - await vi.waitFor(() => expect(mock.evalMock).toHaveBeenCalledTimes(6)); + await vi.waitFor(() => expect(mock.evalMock).toHaveBeenCalledTimes(7)); mock.emit("reconnecting"); resolveOperation?.(["ok", "existing", "generation", "8"]); diff --git a/tests/unit/lib/session-manager-versioned-binding.test.ts b/tests/unit/lib/session-manager-versioned-binding.test.ts index 7078aa0ef..af39e3c7e 100644 --- a/tests/unit/lib/session-manager-versioned-binding.test.ts +++ b/tests/unit/lib/session-manager-versioned-binding.test.ts @@ -9,6 +9,7 @@ const bindingMocks = vi.hoisted(() => ({ refreshSessionBinding: vi.fn(), releaseSessionDiscoveryLease: vi.fn(), renewSessionDiscoveryLease: vi.fn(), + touchSessionBinding: vi.fn(), })); let redisClientRef: { @@ -91,6 +92,10 @@ beforeEach(() => { status: "released", legacyFallbackAllowed: false, }); + bindingMocks.touchSessionBinding.mockResolvedValue({ + ...snapshot(), + source: "touched", + }); }); describe("SessionManager versioned binding adapter", () => { @@ -142,6 +147,28 @@ describe("SessionManager versioned binding adapter", () => { expect(redisClientRef.get).not.toHaveBeenCalled(); }); + it("touches only the captured binding and exposes a TTL-derived heartbeat interval", async () => { + const binding = snapshot().snapshot; + const configuredTtl = Number.parseInt(process.env.SESSION_TTL || "300", 10); + + await expect(SessionManager.touchVersionedSessionBinding(binding)).resolves.toMatchObject({ + status: "ok", + source: "touched", + }); + + expect(bindingMocks.touchSessionBinding).toHaveBeenCalledWith({ + sessionId: SESSION_ID, + keyId: KEY_ID, + expectedGeneration: "generation-a", + expectedProviderId: PROVIDER_ID, + ttlSeconds: configuredTtl, + redis: redisClientRef, + }); + expect(SessionManager.getVersionedSessionBindingRefreshIntervalMs()).toBe( + Math.max(1, Math.floor((configuredTtl * 1000) / 3)) + ); + }); + it("fails closed on a foreign legacy owner", async () => { bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ status: "conflict", From 3d314e3279ab878d0e4e28bd0cc9fc48e78c5ad8 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 20:22:03 -0400 Subject: [PATCH 48/89] fix(discovery): preserve long-stream binding safety --- .../_components/system-settings-form.tsx | 3 +- src/app/v1/_lib/proxy/response-handler.ts | 39 ++++- .../integration/proxy-hedge-lifecycle.test.ts | 12 ++ ...esponse-handler-client-abort-drain.test.ts | 8 ++ ...handler-endpoint-circuit-isolation.test.ts | 135 ++++++++++++++++++ 5 files changed, 193 insertions(+), 4 deletions(-) diff --git a/src/app/[locale]/settings/config/_components/system-settings-form.tsx b/src/app/[locale]/settings/config/_components/system-settings-form.tsx index c887a6d2f..f43859f32 100644 --- a/src/app/[locale]/settings/config/_components/system-settings-form.tsx +++ b/src/app/[locale]/settings/config/_components/system-settings-form.tsx @@ -265,7 +265,8 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) }; if ( discoveryEnabled && - Object.values(discoveryConfig).some((value) => !Number.isSafeInteger(value) || value < 1) + (discoveryConfig.discoveryConcurrency < 2 || + Object.values(discoveryConfig).some((value) => !Number.isSafeInteger(value) || value < 1)) ) { toast.error(t("discoveryWindowInvalid")); return; diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index a9ca3a7de..eee1262a2 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -114,7 +114,8 @@ function isSessionBindingMutationAllowed(session: ProxySession): boolean { } function startDiscoveryLeaseLifecycle(session: ProxySession): DiscoveryLeaseLifecycle { - const lease = peekDeferredStreamingFinalization(session)?.discoveryLease; + const deferred = peekDeferredStreamingFinalization(session); + const lease = deferred?.discoveryLease; if (!lease) { return { active: false, @@ -127,6 +128,12 @@ function startDiscoveryLeaseLifecycle(session: ProxySession): DiscoveryLeaseLife let renewalInFlight: Promise | null = null; let releasePromise: Promise | null = null; let ownershipState: "unknown" | "owned" | "lost" = "unknown"; + const bindingSnapshot = + (deferred?.bindingIntent === "create" || deferred?.bindingIntent === "renew") && + deferred.bindingSnapshot?.sessionId === lease.sessionId && + deferred.bindingSnapshot.keyId === lease.keyId + ? deferred.bindingSnapshot + : null; const stopRenewal = () => { if (renewalTimer) { @@ -157,6 +164,25 @@ function startDiscoveryLeaseLifecycle(session: ProxySession): DiscoveryLeaseLife }); return false; } + + if (bindingSnapshot) { + const touched = await SessionManager.touchVersionedSessionBinding(bindingSnapshot); + if ( + touched.status !== "ok" || + touched.snapshot.generation !== bindingSnapshot.generation || + touched.snapshot.providerId !== bindingSnapshot.providerId + ) { + ownershipState = "lost"; + stopRenewal(); + logger.warn("[ResponseHandler] Discovery binding heartbeat stopped", { + sessionId: lease.sessionId, + keyId: lease.keyId, + status: touched.status, + reason: "reason" in touched ? touched.reason : "snapshot_mismatch", + }); + return false; + } + } ownershipState = "owned"; return true; })() @@ -181,7 +207,14 @@ function startDiscoveryLeaseLifecycle(session: ProxySession): DiscoveryLeaseLife // the downstream response. Renew immediately so an expired/lost token is // observed before any terminal Session binding mutation is attempted. const handoffRenewal = renew(); - const renewalIntervalMs = Math.max(250, Math.floor((lease.ttlSeconds * 1000) / 3)); + const leaseRenewalIntervalMs = Math.floor((lease.ttlSeconds * 1000) / 3); + const bindingRefreshIntervalMs = bindingSnapshot + ? SessionManager.getVersionedSessionBindingRefreshIntervalMs() + : Number.POSITIVE_INFINITY; + const renewalIntervalMs = Math.max( + 250, + Math.min(leaseRenewalIntervalMs, bindingRefreshIntervalMs) + ); renewalTimer = setInterval(() => { void renew(); }, renewalIntervalMs); @@ -1174,7 +1207,7 @@ function hasStreamCompletionMarker(text: string, format: ProxySession["originalF case "claude": return events.some( (event) => - event.event === "message_stop" && + (event.event === "message_stop" || event.event === "message") && isRecord(event.data) && event.data.type === "message_stop" ); diff --git a/tests/integration/proxy-hedge-lifecycle.test.ts b/tests/integration/proxy-hedge-lifecycle.test.ts index 0f2b6b3b0..7707cabb6 100644 --- a/tests/integration/proxy-hedge-lifecycle.test.ts +++ b/tests/integration/proxy-hedge-lifecycle.test.ts @@ -8,6 +8,7 @@ import { ProxyResponseHandler } from "@/app/v1/_lib/proxy/response-handler"; import { type MessageContext, ProxySession } from "@/app/v1/_lib/proxy/session"; import { DbPoolAdmissionError } from "@/drizzle/admitted-client"; import { getGlobalAgentPool, resetGlobalAgentPool } from "@/lib/proxy-agent"; +import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import type { Key } from "@/types/key"; import type { Provider } from "@/types/provider"; import type { User } from "@/types/user"; @@ -125,6 +126,17 @@ vi.mock("@/lib/session-manager", async (importOriginal) => { static override async renewSessionDiscoveryLease() { return state.renewDiscoveryLease(); } + static override getVersionedSessionBindingRefreshIntervalMs() { + return 100_000; + } + static override async touchVersionedSessionBinding(snapshot: SessionBindingSnapshot) { + return { + status: "ok" as const, + source: "touched" as const, + snapshot, + legacyFallbackAllowed: false as const, + }; + } static override async releaseSessionDiscoveryLease() { return state.releaseDiscoveryLease(); } diff --git a/tests/unit/proxy/response-handler-client-abort-drain.test.ts b/tests/unit/proxy/response-handler-client-abort-drain.test.ts index ded21ba4b..55c8ff3b2 100644 --- a/tests/unit/proxy/response-handler-client-abort-drain.test.ts +++ b/tests/unit/proxy/response-handler-client-abort-drain.test.ts @@ -10,6 +10,7 @@ import { AsyncTaskManager, shutdownAllAsyncTasks } from "@/lib/async-task-manage import { recordFailure } from "@/lib/circuit-breaker"; import { emitProxyLangfuseTrace } from "@/lib/langfuse/emit-proxy-trace"; import { RateLimitService } from "@/lib/rate-limit"; +import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import { SessionManager } from "@/lib/session-manager"; import { updateMessageRequestCostWithBreakdown, @@ -118,6 +119,7 @@ vi.mock("@/lib/session-manager", () => ({ clearVersionedSessionProvider: vi.fn(), compareAndSetSessionProvider: vi.fn(), getSessionBindingSnapshot: vi.fn(), + getVersionedSessionBindingRefreshIntervalMs: vi.fn(() => 100_000), renewSessionDiscoveryLease: vi.fn(async () => ({ status: "renewed", legacyFallbackAllowed: false, @@ -126,6 +128,12 @@ vi.mock("@/lib/session-manager", () => ({ status: "released", legacyFallbackAllowed: false, })), + touchVersionedSessionBinding: vi.fn(async (snapshot: SessionBindingSnapshot) => ({ + status: "ok", + source: "touched", + snapshot, + legacyFallbackAllowed: false, + })), extractCodexPromptCacheKey: vi.fn(), storeSessionResponse: vi.fn(async () => undefined), storeSessionRequestPhaseSnapshot: vi.fn(), diff --git a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts index 038c9378c..779241a8f 100644 --- a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts +++ b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts @@ -80,8 +80,10 @@ vi.mock("@/lib/session-manager", () => ({ clearVersionedSessionProvider: vi.fn(), compareAndSetSessionProvider: vi.fn(), getSessionBindingSnapshot: vi.fn(), + getVersionedSessionBindingRefreshIntervalMs: vi.fn(), renewSessionDiscoveryLease: vi.fn(), releaseSessionDiscoveryLease: vi.fn(), + touchVersionedSessionBinding: vi.fn(), extractCodexPromptCacheKey: vi.fn(), updateSessionBindingSmart: vi.fn(), updateSessionProvider: vi.fn(), @@ -468,10 +470,17 @@ function setupCommonMocks() { }, legacyFallbackAllowed: false, }); + vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(100_000); vi.mocked(SessionManager.renewSessionDiscoveryLease).mockResolvedValue({ status: "renewed", legacyFallbackAllowed: false, }); + vi.mocked(SessionManager.touchVersionedSessionBinding).mockImplementation(async (binding) => ({ + status: "ok", + source: "touched", + snapshot: binding, + legacyFallbackAllowed: false, + })); vi.mocked(SessionManager.releaseSessionDiscoveryLease).mockResolvedValue({ status: "released", legacyFallbackAllowed: false, @@ -845,6 +854,11 @@ describe("Endpoint circuit breaker isolation", () => { }); it.each([ + { + label: "Claude data-only stop", + format: "claude" as const, + body: `data: ${JSON.stringify({ type: "message_stop" })}\n\n`, + }, { label: "OpenAI Responses", format: "response" as const, @@ -915,6 +929,7 @@ describe("Endpoint circuit breaker isolation", () => { await drainAsyncTasks(); expect(SessionManager.compareAndSetSessionProvider).toHaveBeenCalledWith(snapshot, 1); + expect(mockRecordFailure).not.toHaveBeenCalled(); }); it("releases a create attempt ref when generation CAS loses", async () => { @@ -1165,6 +1180,126 @@ describe("Endpoint circuit breaker isolation", () => { } }); + it("touches the captured binding often enough for a long Discovery winner", async () => { + vi.useFakeTimers(); + try { + const session = createSession(); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "long-stream-generation", + } as const; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: snapshot, + requiresCompletionMarker: true, + discoveryLease: { + sessionId: "fake-session", + keyId: 456, + ownerToken: "long-stream-owner", + ttlSeconds: 3_600, + }, + }); + vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(1_000); + const controlled = createControllableSuccessStreamResponse(); + + const clientResponse = await ProxyResponseHandler.dispatch(session, controlled.response); + await vi.advanceTimersByTimeAsync(3_000); + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes(4); + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenLastCalledWith(snapshot); + + controlled.complete(); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.compareAndSetSessionProvider).toHaveBeenCalledWith(snapshot, 1); + expect(SessionManager.getSessionBindingSnapshot).not.toHaveBeenCalled(); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledWith( + "fake-session", + 456, + "long-stream-owner" + ); + } finally { + vi.useRealTimers(); + } + }); + + it("does not revive a binding after an authority advances its generation", async () => { + vi.useFakeTimers(); + try { + const session = createSession(); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: 1, + generation: "generation-before-termination", + } as const; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 1, + isFirstAttempt: true, + isFailoverSuccess: false, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "renew", + bindingSnapshot: snapshot, + requiresCompletionMarker: true, + discoveryLease: { + sessionId: "fake-session", + keyId: 456, + ownerToken: "terminated-stream-owner", + ttlSeconds: 3_600, + }, + }); + vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(1_000); + vi.mocked(SessionManager.touchVersionedSessionBinding) + .mockResolvedValueOnce({ + status: "ok", + source: "touched", + snapshot, + legacyFallbackAllowed: false, + }) + .mockResolvedValueOnce({ + status: "conflict", + reason: "generation_mismatch", + legacyFallbackAllowed: false, + }); + const controlled = createControllableSuccessStreamResponse(); + + const clientResponse = await ProxyResponseHandler.dispatch(session, controlled.response); + await vi.advanceTimersByTimeAsync(1_000); + controlled.complete(); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes(2); + expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.getSessionBindingSnapshot).not.toHaveBeenCalled(); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledWith( + "fake-session", + 456, + "terminated-stream-owner" + ); + } finally { + vi.useRealTimers(); + } + }); + it("does not delay downstream delivery while the lease handoff renewal is pending", async () => { const handoffRenewal = Promise.withResolvers<{ status: "renewed"; From 825f4ff380cdd301dd168caa21d1a34473012216 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 20:42:27 -0400 Subject: [PATCH 49/89] fix(discovery): fail closed on binding conflicts --- src/app/v1/_lib/proxy/forwarder.ts | 10 ++++-- src/app/v1/_lib/proxy/response-handler.ts | 14 ++++---- .../proxy-forwarder-hedge-first-byte.test.ts | 33 +++++++++++++------ ...handler-endpoint-circuit-isolation.test.ts | 13 ++++++++ 4 files changed, 50 insertions(+), 20 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 1f1489c00..3012b1907 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -3945,9 +3945,13 @@ export class ProxyForwarder { const binding = await SessionManager.getSessionBindingSnapshot(sessionId, keyId); if (binding.status !== "ok") { // A foreign or irreconcilable mirror must never be mutated by this - // request. Infrastructure unavailability still falls back to the - // established legacy wrapper. - if (binding.status === "conflict") session.setSessionBindingAllowed(false); + // request or fan out through legacy Hedge. Explicit upstream failures + // may still use the established one-at-a-time provider fallback below. + // Infrastructure unavailability continues to use the legacy wrapper. + if (binding.status === "conflict") { + session.disableStreamingHedge(); + session.setSessionBindingAllowed(false); + } return null; } bindingSnapshot = binding.snapshot; diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index eee1262a2..aa66e78c2 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -1197,13 +1197,13 @@ function hasStreamCompletionMarker(text: string, format: ProxySession["originalF switch (format) { case "response": - return events.some( - (event) => - event.event === "response.completed" && - isRecord(event.data) && - event.data.type === "response.completed" && - isRecord(event.data.response) - ); + return events.some((event) => { + if (!isRecord(event.data)) return false; + const markerType = event.data.type; + if (markerType !== "response.completed" && markerType !== "response.done") return false; + if (event.event !== "message" && event.event !== markerType) return false; + return markerType === "response.done" || isRecord(event.data.response); + }); case "claude": return events.some( (event) => diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index 7bbcc1856..91be7ff5d 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -168,6 +168,7 @@ import { import { ProxyForwarder } from "@/app/v1/_lib/proxy/forwarder"; import { ModelRedirector } from "@/app/v1/_lib/proxy/model-redirector"; import { ProxySession } from "@/app/v1/_lib/proxy/session"; +import { peekDeferredStreamingFinalization } from "@/app/v1/_lib/proxy/stream-finalization"; import { DbPoolAdmissionError } from "@/drizzle/admitted-client"; import { logger } from "@/lib/logger"; import type { Provider } from "@/types/provider"; @@ -2362,8 +2363,9 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { expect(mocks.getCachedSystemSettings).toHaveBeenCalledTimes(1); }); - test("foreign binding state fails closed before Discovery acquires a lease", async () => { - const provider = createProvider({ id: 1, firstByteTimeoutStreamingMs: 0 }); + test("foreign binding state uses single-upstream routing with serial fallback", async () => { + const provider = createProvider({ id: 1, firstByteTimeoutStreamingMs: 100 }); + const alternative = createProvider({ id: 2, name: "serial-fallback" }); const session = createSession(); session.authState = { success: true, @@ -2378,6 +2380,7 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { reason: "legacy_owner_mismatch", legacyFallbackAllowed: false, }); + mocks.pickRandomProviderWithExclusion.mockResolvedValueOnce(alternative); const doForward = vi.spyOn( ProxyForwarder as unknown as { @@ -2385,16 +2388,26 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { }, "doForward" ); - doForward.mockResolvedValueOnce( - new Response('data: {"type":"message_stop"}\n\n', { - status: 200, - headers: { "content-type": "text/event-stream" }, - }) - ); + doForward + .mockRejectedValueOnce(new UpstreamProxyError("initial provider failed", 500)) + .mockResolvedValueOnce( + new Response('data: {"type":"message_stop"}\n\n', { + status: 200, + headers: { "content-type": "text/event-stream" }, + }) + ); - await ProxyForwarder.send(session); - expect(doForward).toHaveBeenCalledTimes(1); + const response = await ProxyForwarder.send(session); + expect(response.status).toBe(200); + expect(doForward).toHaveBeenCalledTimes(2); + expect(doForward.mock.calls.map((call) => (call[1] as Provider).id)).toEqual([ + provider.id, + alternative.id, + ]); + expect(mocks.pickRandomProviderWithExclusion).toHaveBeenCalledTimes(1); + expect(session.isStreamingHedgeDisabled()).toBe(true); expect(session.isSessionBindingAllowed()).toBe(false); + expect(peekDeferredStreamingFinalization(session)?.bindingIntent).toBe("none"); expect(mocks.acquireSessionDiscoveryLease).not.toHaveBeenCalled(); expect(mocks.pickDiscoveryProviders).not.toHaveBeenCalled(); }); diff --git a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts index 779241a8f..e076c2d8f 100644 --- a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts +++ b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts @@ -867,6 +867,19 @@ describe("Endpoint circuit breaker isolation", () => { response: { id: "resp_completed" }, })}\n\n`, }, + { + label: "OpenAI Responses data-only completed", + format: "response" as const, + body: `data: ${JSON.stringify({ + type: "response.completed", + response: { id: "resp_data_only_completed" }, + })}\n\n`, + }, + { + label: "OpenAI Responses done", + format: "response" as const, + body: `event: response.done\ndata: ${JSON.stringify({ type: "response.done" })}\n\n`, + }, { label: "OpenAI Chat finish reason", format: "openai" as const, From ef6b833f12d490fa77046ffbfec20d6e5fc16b89 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 20:43:27 -0400 Subject: [PATCH 50/89] fix(discovery): close final lifecycle gaps --- src/app/v1/_lib/proxy/forwarder.ts | 66 ++++--- src/app/v1/_lib/proxy/response-handler.ts | 55 +++++- .../integration/proxy-hedge-lifecycle.test.ts | 12 ++ .../proxy-forwarder-hedge-first-byte.test.ts | 172 ++++++++++++++++++ ...esponse-handler-client-abort-drain.test.ts | 65 +++++++ ...handler-endpoint-circuit-isolation.test.ts | 123 ++++++++++++- 6 files changed, 462 insertions(+), 31 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 1f1489c00..ffaf62d68 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -5167,7 +5167,7 @@ export class ProxyForwarder { cancelLosers(null, options.cancellationKind ?? "discovery_loser"); if (!options.preserveBinding && bindingWriteAllowed && session.isSessionBindingAllowed()) { if (bindingSnapshot.providerId != null) { - void SessionManager.clearVersionedSessionProvider( + await SessionManager.clearVersionedSessionProvider( bindingSnapshot, bindingSnapshot.providerId, 0 @@ -5362,7 +5362,7 @@ export class ProxyForwarder { retainOnSuccess: boolean; }; } - ): Promise => { + ): Promise => { const transferredProviderSessionRef = options?.providerSessionRefTransfer?.owned === true; let providerSessionRefTracked = transferredProviderSessionRef; let providerSessionRefRetainOnSuccess = @@ -5376,7 +5376,7 @@ export class ProxyForwarder { }; if (settled || committed || launched.has(provider.id)) { rollbackLaunch(); - return; + return false; } launched.add(provider.id); if (!transferredProviderSessionRef && provider.id === initialProvider.id) { @@ -5404,7 +5404,7 @@ export class ProxyForwarder { } if (settled || committed) { rollbackLaunch(); - return; + return false; } let endpoint: Awaited>; try { @@ -5415,7 +5415,7 @@ export class ProxyForwarder { } if (settled || committed) { rollbackLaunch(); - return; + return false; } let attemptSession: ProxySession; try { @@ -5431,7 +5431,7 @@ export class ProxyForwarder { } if (settled || committed) { rollbackLaunch(); - return; + return false; } const controller = new AbortController(); const id = `${provider.id}:${sequence + 1}`; @@ -5505,7 +5505,7 @@ export class ProxyForwarder { }); if (!registered || settled || committed) { rollbackLaunch(); - return; + return false; } attempts.set(id, attempt); discoveryMetrics.attemptStarted({ @@ -5821,6 +5821,7 @@ export class ProxyForwarder { error, }); }); + return true; }; const launchNextRound = async (slots: number, coordinatorAlreadyAdvanced = false) => { @@ -5835,21 +5836,42 @@ export class ProxyForwarder { currentRound = nextRound.round; } if (currentRound > maxRounds || slots <= 0) return; - const candidates = await ProxyProviderResolver.pickDiscoveryProviders( - session, - slots, - Array.from(launched) - ); - if (candidates.length === 0) { - noMoreCandidates = true; - } - for (const candidate of candidates) { - try { - await launch(candidate, "normal"); - } catch (error) { - lastError = error instanceof Error ? error : new Error(String(error)); - // Launch setup failures do not establish pool exhaustion. - noMoreCandidates = false; + let remainingSlots = slots; + while (remainingSlots > 0 && !settled && !committed) { + const exclusionCountBeforeSelection = launched.size; + const candidates = await ProxyProviderResolver.pickDiscoveryProviders( + session, + remainingSlots, + Array.from(launched) + ); + if (candidates.length === 0) { + noMoreCandidates = true; + break; + } + + noMoreCandidates = false; + let registeredInBatch = 0; + for (const candidate of candidates) { + try { + if (await launch(candidate, "normal")) { + registeredInBatch += 1; + remainingSlots -= 1; + } + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + // This Provider is already excluded by launch(). Continue filling + // the same round from the remaining candidate pool. + noMoreCandidates = false; + } + if (remainingSlots <= 0 || settled || committed) break; + } + + // A selector that ignores exclusions must not create an unbounded + // setup loop. Normal selectors always advance `launched`, including + // attempts that fail before transport registration. + if (registeredInBatch === 0 && launched.size === exclusionCountBeforeSelection) { + noMoreCandidates = true; + break; } } const hasPendingAttempt = Array.from(attempts.values()).some((attempt) => attempt.pending); diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index a9ca3a7de..064483a37 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -114,7 +114,8 @@ function isSessionBindingMutationAllowed(session: ProxySession): boolean { } function startDiscoveryLeaseLifecycle(session: ProxySession): DiscoveryLeaseLifecycle { - const lease = peekDeferredStreamingFinalization(session)?.discoveryLease; + const deferred = peekDeferredStreamingFinalization(session); + const lease = deferred?.discoveryLease; if (!lease) { return { active: false, @@ -127,6 +128,12 @@ function startDiscoveryLeaseLifecycle(session: ProxySession): DiscoveryLeaseLife let renewalInFlight: Promise | null = null; let releasePromise: Promise | null = null; let ownershipState: "unknown" | "owned" | "lost" = "unknown"; + const bindingSnapshot = + (deferred?.bindingIntent === "create" || deferred?.bindingIntent === "renew") && + deferred.bindingSnapshot?.sessionId === lease.sessionId && + deferred.bindingSnapshot.keyId === lease.keyId + ? deferred.bindingSnapshot + : null; const stopRenewal = () => { if (renewalTimer) { @@ -157,6 +164,25 @@ function startDiscoveryLeaseLifecycle(session: ProxySession): DiscoveryLeaseLife }); return false; } + + if (bindingSnapshot) { + const touched = await SessionManager.touchVersionedSessionBinding(bindingSnapshot); + if ( + touched.status !== "ok" || + touched.snapshot.generation !== bindingSnapshot.generation || + touched.snapshot.providerId !== bindingSnapshot.providerId + ) { + ownershipState = "lost"; + stopRenewal(); + logger.warn("[ResponseHandler] Discovery binding heartbeat stopped", { + sessionId: lease.sessionId, + keyId: lease.keyId, + status: touched.status, + reason: "reason" in touched ? touched.reason : "snapshot_mismatch", + }); + return false; + } + } ownershipState = "owned"; return true; })() @@ -181,7 +207,14 @@ function startDiscoveryLeaseLifecycle(session: ProxySession): DiscoveryLeaseLife // the downstream response. Renew immediately so an expired/lost token is // observed before any terminal Session binding mutation is attempted. const handoffRenewal = renew(); - const renewalIntervalMs = Math.max(250, Math.floor((lease.ttlSeconds * 1000) / 3)); + const leaseRenewalIntervalMs = Math.floor((lease.ttlSeconds * 1000) / 3); + const bindingRefreshIntervalMs = bindingSnapshot + ? SessionManager.getVersionedSessionBindingRefreshIntervalMs() + : Number.POSITIVE_INFINITY; + const renewalIntervalMs = Math.max( + 250, + Math.min(leaseRenewalIntervalMs, bindingRefreshIntervalMs) + ); renewalTimer = setInterval(() => { void renew(); }, renewalIntervalMs); @@ -1468,6 +1501,10 @@ function finalizeDeferredStreamingFinalizationIfNeeded( primaryDiscoveryBindingSettled = true; resolvePrimaryDiscoveryBinding?.(updated); }; + const finalizeFailedDiscoveryBinding = async () => { + settlePrimaryDiscoveryBinding(false); + await finalizeProviderSessionRef(); + }; const confirmAuxiliarySessionBinding = async () => allowAuxiliarySessionBinding && (await primaryDiscoveryBinding); @@ -1569,12 +1606,14 @@ function finalizeDeferredStreamingFinalizationIfNeeded( // - 不在这里更新熔断/绑定（meta 缺失意味着 Forwarder 没有启用延迟结算；provider 缺失意味着无法归因）。 if (!meta || !provider) { const commitSideEffects = - shouldClearSessionBindingOnFailure || meta?.providerSessionRefOwned === true + shouldClearSessionBindingOnFailure || + meta?.providerSessionRefOwned === true || + hasDiscoveryBindingIntent ? async () => { try { if (shouldClearSessionBindingOnFailure) await clearSessionBinding(); } finally { - await finalizeProviderSessionRef(); + await finalizeFailedDiscoveryBinding(); } } : undefined; @@ -1651,7 +1690,7 @@ function finalizeDeferredStreamingFinalizationIfNeeded( // so only the Provider circuit is updated here. } } finally { - await finalizeProviderSessionRef(); + await finalizeFailedDiscoveryBinding(); } }; @@ -1695,7 +1734,7 @@ function finalizeDeferredStreamingFinalizationIfNeeded( } } } finally { - await finalizeProviderSessionRef(); + await finalizeFailedDiscoveryBinding(); } }; @@ -1766,7 +1805,7 @@ function finalizeDeferredStreamingFinalizationIfNeeded( } } } finally { - await finalizeProviderSessionRef(); + await finalizeFailedDiscoveryBinding(); } }; @@ -1830,7 +1869,7 @@ function finalizeDeferredStreamingFinalizationIfNeeded( } } } finally { - await finalizeProviderSessionRef(); + await finalizeFailedDiscoveryBinding(); } }; diff --git a/tests/integration/proxy-hedge-lifecycle.test.ts b/tests/integration/proxy-hedge-lifecycle.test.ts index 0f2b6b3b0..7707cabb6 100644 --- a/tests/integration/proxy-hedge-lifecycle.test.ts +++ b/tests/integration/proxy-hedge-lifecycle.test.ts @@ -8,6 +8,7 @@ import { ProxyResponseHandler } from "@/app/v1/_lib/proxy/response-handler"; import { type MessageContext, ProxySession } from "@/app/v1/_lib/proxy/session"; import { DbPoolAdmissionError } from "@/drizzle/admitted-client"; import { getGlobalAgentPool, resetGlobalAgentPool } from "@/lib/proxy-agent"; +import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import type { Key } from "@/types/key"; import type { Provider } from "@/types/provider"; import type { User } from "@/types/user"; @@ -125,6 +126,17 @@ vi.mock("@/lib/session-manager", async (importOriginal) => { static override async renewSessionDiscoveryLease() { return state.renewDiscoveryLease(); } + static override getVersionedSessionBindingRefreshIntervalMs() { + return 100_000; + } + static override async touchVersionedSessionBinding(snapshot: SessionBindingSnapshot) { + return { + status: "ok" as const, + source: "touched" as const, + snapshot, + legacyFallbackAllowed: false as const, + }; + } static override async releaseSessionDiscoveryLease() { return state.releaseDiscoveryLease(); } diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index 7bbcc1856..bec72c795 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -2848,6 +2848,101 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { } }); + test("Sticky Discovery refills a slot immediately after candidate setup fails", async () => { + vi.useFakeTimers(); + let endpointResolver: ReturnType | null = null; + try { + const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); + const setupFailure = createProvider({ id: 2, name: "setup-failure", priority: 1 }); + const replacement = createProvider({ id: 3, name: "replacement", priority: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 30 }, + apiKey: null, + } as typeof session.authState; + session.request.message.messages = [ + { role: "user", content: "first" }, + { role: "user", content: "second" }, + ]; + session.setProvider(sticky); + session.setSessionBindingSnapshot({ + sessionId: session.sessionId!, + keyId: 30, + providerId: sticky.id, + generation: "setup-refill-generation", + }); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 50, + stickySlaMs: 10, + racingTotalTimeoutMs: 200, + stickyTimeoutCooldownMs: 300_000, + }); + mocks.pickDiscoveryProviders + .mockResolvedValueOnce([setupFailure]) + .mockResolvedValueOnce([replacement]); + + endpointResolver = vi.spyOn( + ProxyForwarder as unknown as { + resolveStreamingHedgeEndpoint: ( + session: ProxySession, + provider: Provider + ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + }, + "resolveStreamingHedgeEndpoint" + ); + endpointResolver.mockImplementation(async (_attemptSession, provider) => { + if (provider.id === setupFailure.id) throw new Error("candidate endpoint setup failed"); + return { + endpointId: null, + baseUrl: provider.url, + endpointUrl: provider.url, + }; + }); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockImplementation(async (attemptSession) => { + if ((attemptSession as ProxySession).provider?.id === sticky.id) { + return new Response(new ReadableStream(), { + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response( + 'data: {"type":"content_block_delta","delta":{"text":"replacement"}}\n\n', + { headers: { "content-type": "text/event-stream" } } + ); + }); + + const responsePromise = ProxyForwarder.send(session); + await vi.advanceTimersByTimeAsync(10); + await vi.advanceTimersByTimeAsync(0); + const response = await responsePromise; + + expect(await response.text()).toContain('"replacement"'); + expect(mocks.pickDiscoveryProviders).toHaveBeenCalledTimes(2); + expect(mocks.pickDiscoveryProviders).toHaveBeenNthCalledWith( + 2, + expect.anything(), + 1, + expect.arrayContaining([sticky.id, setupFailure.id]) + ); + expect(doForward).toHaveBeenCalledTimes(2); + expect(session.provider?.id).toBe(replacement.id); + } finally { + endpointResolver?.mockRestore(); + vi.useRealTimers(); + } + }); + test("an explicit Sticky failure starts Discovery round one at full concurrency", async () => { const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); const normal = createProvider({ id: 2, name: "normal", priority: 1 }); @@ -2976,6 +3071,83 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { expect(mocks.releaseSessionDiscoveryLease).toHaveBeenCalledTimes(1); }); + test("Discovery clears terminal binding state before releasing its lease", async () => { + const provider = createProvider({ id: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 29 }, + apiKey: null, + } as typeof session.authState; + session.setProvider(provider); + session.setSessionBindingSnapshot({ + sessionId: session.sessionId!, + keyId: 29, + providerId: provider.id, + generation: "terminal-clear-generation", + }); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 100, + stickySlaMs: 100, + racingTotalTimeoutMs: 500, + }); + const order: string[] = []; + const clear = Promise.withResolvers<{ + status: "ok"; + legacyFallbackAllowed: false; + source: "cleared"; + snapshot: { + sessionId: string; + keyId: number; + providerId: null; + generation: string; + }; + }>(); + mocks.clearVersionedSessionProvider.mockImplementationOnce(async () => { + order.push("clear-start"); + const result = await clear.promise; + order.push("clear-end"); + return result; + }); + mocks.releaseSessionDiscoveryLease.mockImplementationOnce(async () => { + order.push("lease-release"); + return { status: "released", legacyFallbackAllowed: false }; + }); + vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ).mockRejectedValueOnce(new Error("terminal upstream failure")); + + const observed = ProxyForwarder.send(session).catch((error) => error); + for (let index = 0; index < 10 && order.length === 0; index++) { + await new Promise((resolve) => setImmediate(resolve)); + } + expect(order).toEqual(["clear-start"]); + expect(mocks.releaseSessionDiscoveryLease).not.toHaveBeenCalled(); + + clear.resolve({ + status: "ok", + legacyFallbackAllowed: false, + source: "cleared", + snapshot: { + sessionId: session.sessionId!, + keyId: 29, + providerId: null, + generation: "terminal-cleared-generation", + }, + }); + expect(await observed).toBeInstanceOf(Error); + expect(mocks.clearVersionedSessionProvider).toHaveBeenCalledOnce(); + expect(mocks.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); + expect(order).toEqual(["clear-start", "clear-end", "lease-release"]); + }); + test("Discovery total deadline is not blocked by a stalled candidate selector", async () => { vi.useFakeTimers(); try { diff --git a/tests/unit/proxy/response-handler-client-abort-drain.test.ts b/tests/unit/proxy/response-handler-client-abort-drain.test.ts index ded21ba4b..d52cb370e 100644 --- a/tests/unit/proxy/response-handler-client-abort-drain.test.ts +++ b/tests/unit/proxy/response-handler-client-abort-drain.test.ts @@ -10,6 +10,7 @@ import { AsyncTaskManager, shutdownAllAsyncTasks } from "@/lib/async-task-manage import { recordFailure } from "@/lib/circuit-breaker"; import { emitProxyLangfuseTrace } from "@/lib/langfuse/emit-proxy-trace"; import { RateLimitService } from "@/lib/rate-limit"; +import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import { SessionManager } from "@/lib/session-manager"; import { updateMessageRequestCostWithBreakdown, @@ -118,6 +119,7 @@ vi.mock("@/lib/session-manager", () => ({ clearVersionedSessionProvider: vi.fn(), compareAndSetSessionProvider: vi.fn(), getSessionBindingSnapshot: vi.fn(), + getVersionedSessionBindingRefreshIntervalMs: vi.fn(() => 100_000), renewSessionDiscoveryLease: vi.fn(async () => ({ status: "renewed", legacyFallbackAllowed: false, @@ -126,6 +128,12 @@ vi.mock("@/lib/session-manager", () => ({ status: "released", legacyFallbackAllowed: false, })), + touchVersionedSessionBinding: vi.fn(async (snapshot: SessionBindingSnapshot) => ({ + status: "ok", + source: "touched", + snapshot, + legacyFallbackAllowed: false, + })), extractCodexPromptCacheKey: vi.fn(), storeSessionResponse: vi.fn(async () => undefined), storeSessionRequestPhaseSnapshot: vi.fn(), @@ -3473,6 +3481,63 @@ describe("ProxyResponseHandler stream client abort finalization", () => { expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); }); + it("settles a failed Discovery binding before rejecting its auxiliary Codex cache binding", async () => { + vi.mocked(SessionManager.extractCodexPromptCacheKey).mockReturnValueOnce( + "incomplete-discovery-cache-key" + ); + const session = createSession(new AbortController().signal); + session.sessionId = "stream-discovery-cache-incomplete"; + session.recordProviderSessionRef(1); + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "avemujica-responses", + providerPriority: 1, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.invalid/v1", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: { + sessionId: "stream-discovery-cache-incomplete", + keyId: 2, + providerId: null, + generation: "incomplete-discovery-generation", + }, + requiresCompletionMarker: true, + discoveryLease: { + sessionId: "stream-discovery-cache-incomplete", + keyId: 2, + ownerToken: "incomplete-discovery-owner", + ttlSeconds: 30, + }, + providerSessionRefOwned: true, + }); + const incomplete = new Response( + `event: response.output_text.done\ndata: ${JSON.stringify({ + type: "response.output_text.done", + text: "partial", + })}\n\n`, + { status: 200, headers: { "content-type": "text/event-stream" } } + ); + + const downstream = await ProxyResponseHandler.dispatch(session, incomplete); + await downstream.text(); + for (let index = 0; index < 10 && !getRegisteredTask("post-terminal-side-effects"); index++) { + await new Promise((resolve) => setImmediate(resolve)); + } + const sideEffects = getRegisteredTask("post-terminal-side-effects"); + expect(sideEffects).toBeDefined(); + await expectTaskToResolveWithoutWaiting(sideEffects as Promise); + await drainAsyncTasks(); + + expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.updateSessionWithCodexCacheKey).not.toHaveBeenCalled(); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); + }); + it("does not publish a stream Codex cache binding for a final non-2xx outcome", async () => { vi.mocked(SessionManager.extractCodexPromptCacheKey).mockReturnValueOnce("stream-cache-key-2"); const session = createSession(new AbortController().signal); diff --git a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts index 038c9378c..c6c664583 100644 --- a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts +++ b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts @@ -80,8 +80,10 @@ vi.mock("@/lib/session-manager", () => ({ clearVersionedSessionProvider: vi.fn(), compareAndSetSessionProvider: vi.fn(), getSessionBindingSnapshot: vi.fn(), + getVersionedSessionBindingRefreshIntervalMs: vi.fn(), renewSessionDiscoveryLease: vi.fn(), releaseSessionDiscoveryLease: vi.fn(), + touchVersionedSessionBinding: vi.fn(), extractCodexPromptCacheKey: vi.fn(), updateSessionBindingSmart: vi.fn(), updateSessionProvider: vi.fn(), @@ -397,7 +399,7 @@ function createControllableSuccessStreamResponse(): { }), complete: () => { streamController.enqueue( - encoder.encode(`data: ${JSON.stringify({ type: "message_stop" })}\n\n`) + encoder.encode(`event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n`) ); streamController.close(); }, @@ -468,10 +470,17 @@ function setupCommonMocks() { }, legacyFallbackAllowed: false, }); + vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(100_000); vi.mocked(SessionManager.renewSessionDiscoveryLease).mockResolvedValue({ status: "renewed", legacyFallbackAllowed: false, }); + vi.mocked(SessionManager.touchVersionedSessionBinding).mockImplementation(async (binding) => ({ + status: "ok", + source: "touched", + snapshot: binding, + legacyFallbackAllowed: false, + })); vi.mocked(SessionManager.releaseSessionDiscoveryLease).mockResolvedValue({ status: "released", legacyFallbackAllowed: false, @@ -1165,6 +1174,118 @@ describe("Endpoint circuit breaker isolation", () => { } }); + it("touches the captured binding while a Discovery winner remains open", async () => { + vi.useFakeTimers(); + try { + const session = createSession(); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "long-stream-generation", + } as const; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: snapshot, + requiresCompletionMarker: true, + discoveryLease: { + sessionId: "fake-session", + keyId: 456, + ownerToken: "long-stream-owner", + ttlSeconds: 3_600, + }, + }); + vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(1_000); + const controlled = createControllableSuccessStreamResponse(); + + const clientResponse = await ProxyResponseHandler.dispatch(session, controlled.response); + await vi.advanceTimersByTimeAsync(3_000); + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes(4); + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenLastCalledWith(snapshot); + + controlled.complete(); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.compareAndSetSessionProvider).toHaveBeenCalledWith(snapshot, 1); + expect(SessionManager.getSessionBindingSnapshot).not.toHaveBeenCalled(); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); + } finally { + vi.useRealTimers(); + } + }); + + it("revokes Sticky writes when a binding heartbeat loses its generation", async () => { + vi.useFakeTimers(); + try { + const session = createSession(); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: 1, + generation: "generation-before-conflict", + } as const; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 1, + isFirstAttempt: true, + isFailoverSuccess: false, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "renew", + bindingSnapshot: snapshot, + requiresCompletionMarker: true, + discoveryLease: { + sessionId: "fake-session", + keyId: 456, + ownerToken: "conflicted-stream-owner", + ttlSeconds: 3_600, + }, + }); + vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(1_000); + vi.mocked(SessionManager.touchVersionedSessionBinding) + .mockResolvedValueOnce({ + status: "ok", + source: "touched", + snapshot, + legacyFallbackAllowed: false, + }) + .mockResolvedValueOnce({ + status: "conflict", + reason: "generation_mismatch", + legacyFallbackAllowed: false, + }); + const controlled = createControllableSuccessStreamResponse(); + + const clientResponse = await ProxyResponseHandler.dispatch(session, controlled.response); + await vi.advanceTimersByTimeAsync(1_000); + controlled.complete(); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes(2); + expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.getSessionBindingSnapshot).not.toHaveBeenCalled(); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); + } finally { + vi.useRealTimers(); + } + }); + it("does not delay downstream delivery while the lease handoff renewal is pending", async () => { const handoffRenewal = Promise.withResolvers<{ status: "renewed"; From 9391c30461dacc60ad158fb45280cd75aaec02a5 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 20:54:59 -0400 Subject: [PATCH 51/89] fix(binding): close long-stream and terminate races --- src/app/v1/_lib/proxy/forwarder.ts | 17 +- src/app/v1/_lib/proxy/response-handler.ts | 118 ++++++++++++ src/app/v1/_lib/proxy/stream-finalization.ts | 10 + src/lib/redis/session-binding.ts | 28 ++- src/lib/session-manager.ts | 10 +- tests/unit/lib/redis/session-binding.test.ts | 84 ++++++++ .../lib/session-manager-binding-smart.test.ts | 6 + .../proxy-forwarder-hedge-first-byte.test.ts | 20 +- ...handler-endpoint-circuit-isolation.test.ts | 181 +++++++++++++++++- 9 files changed, 466 insertions(+), 8 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 632807ef2..042aa9f0c 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -30,6 +30,7 @@ import { } from "@/lib/provider-endpoints/endpoint-selector"; import { getGlobalAgentPool, getProxyAgentForProvider } from "@/lib/proxy-agent"; import { RateLimitService } from "@/lib/rate-limit/service"; +import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import { SessionManager } from "@/lib/session-manager"; import { detectUpstreamErrorFromSseOrJsonText, @@ -4557,8 +4558,9 @@ export class ProxyForwarder { // A non-hedged request is finalized through response-handler. Updating // here as well would perform a duplicate binding read/CAS before the // stream has passed its final validation. + let hedgeBindingSnapshotPromise: Promise | undefined; if (session.sessionId && isActualHedgeWin) { - void (async () => { + hedgeBindingSnapshotPromise = (async () => { const bindingResult = await SessionManager.updateSessionBindingSmart( session.sessionId!, attempt.provider.id, @@ -4581,15 +4583,25 @@ export class ProxyForwarder { } if (session.shouldTrackSessionObservability()) { - await SessionManager.updateSessionProvider(session.sessionId!, { + void SessionManager.updateSessionProvider(session.sessionId!, { providerId: attempt.provider.id, providerName: attempt.provider.name, + }).catch((observabilityError) => { + logger.error( + "ProxyForwarder: Failed to update observable session provider for hedge winner", + { error: observabilityError } + ); }); } + + // Only the exact snapshot returned by the first-byte CAS may keep + // this binding alive. Legacy fallback and failed CAS paths return null. + return bindingResult.bindingSnapshot ?? null; })().catch((bindingError) => { logger.error("ProxyForwarder: Failed to update session provider info for hedge winner", { error: bindingError, }); + return null; }); } @@ -4606,6 +4618,7 @@ export class ProxyForwarder { upstreamStatusCode: attempt.response.status, isHedgeWinner: isActualHedgeWin, billHedgeLosers, + hedgeBindingSnapshotPromise, }); const response = new Response( diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index acd5eb26f..c87fd6ad9 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -58,6 +58,7 @@ import { isClientAbortError, isTransportError } from "./errors"; import type { ProxySession } from "./session"; import { consumeDeferredStreamingFinalization, + type DeferredStreamingBindingHeartbeat, peekDeferredStreamingFinalization, } from "./stream-finalization"; @@ -97,6 +98,96 @@ const STREAM_FINALIZATION_MAX_MS = 120_000; const STREAM_FAILURE_PERSISTENCE_MAX_MS = 5_000; const NON_STREAM_TERMINAL_PERSISTENCE_ERROR = Symbol("non_stream_terminal_persistence_error"); +function startHedgeBindingHeartbeat(session: ProxySession): void { + const deferred = peekDeferredStreamingFinalization(session); + const snapshotPromise = deferred?.hedgeBindingSnapshotPromise; + if (!deferred?.isHedgeWinner || !snapshotPromise || deferred.hedgeBindingHeartbeat) return; + + let periodicActive = true; + let authorityLost = false; + let timer: ReturnType | null = null; + let touchInFlight: Promise | null = null; + let completionPromise: Promise | null = null; + + const stopPeriodic = () => { + periodicActive = false; + if (timer) { + clearInterval(timer); + timer = null; + } + }; + + const loseAuthority = (status: string, reason?: string) => { + if (authorityLost) return; + authorityLost = true; + stopPeriodic(); + logger.warn("[ResponseHandler] Hedge binding heartbeat stopped", { + sessionId: session.sessionId, + status, + reason, + }); + }; + + const touch = (allowAfterStop = false): Promise => { + if (authorityLost || (!periodicActive && !allowAfterStop)) return Promise.resolve(false); + if (touchInFlight) return touchInFlight; + + const operation = (async () => { + const snapshot = await snapshotPromise; + if (!snapshot) { + authorityLost = true; + stopPeriodic(); + return false; + } + if (authorityLost || (!periodicActive && !allowAfterStop)) return false; + + const touched = await SessionManager.touchVersionedSessionBinding(snapshot); + if ( + touched.status !== "ok" || + touched.snapshot.generation !== snapshot.generation || + touched.snapshot.providerId !== snapshot.providerId + ) { + loseAuthority(touched.status, "reason" in touched ? touched.reason : "snapshot_mismatch"); + return false; + } + return true; + })() + .catch((error) => { + loseAuthority("error", error instanceof Error ? error.message : String(error)); + return false; + }) + .finally(() => { + if (touchInFlight === operation) touchInFlight = null; + }); + touchInFlight = operation; + return operation; + }; + + const lifecycle: DeferredStreamingBindingHeartbeat = { + stop: stopPeriodic, + complete: () => { + if (completionPromise) return completionPromise; + stopPeriodic(); + completionPromise = (async () => { + if (touchInFlight) await touchInFlight; + if (authorityLost) return; + await touch(true); + })(); + return completionPromise; + }, + }; + deferred.hedgeBindingHeartbeat = lifecycle; + + // The first touch validates that ownership really transferred with the + // first-byte CAS. Subsequent touches keep streams longer than SESSION_TTL alive. + void touch(); + const intervalMs = Math.max(250, SessionManager.getVersionedSessionBindingRefreshIntervalMs()); + timer = setInterval(() => { + void touch(); + }, intervalMs); + timer.unref?.(); +} + type MessageRequestTerminalDetails = Parameters[1]; type NonStreamTerminalPersistenceError = Error & { [NON_STREAM_TERMINAL_PERSISTENCE_ERROR]: true; @@ -1149,6 +1240,21 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const providerIdForPersistence = meta?.providerId ?? provider?.id ?? null; const clearSessionBinding = async () => { if (!session.sessionId) return; + const hedgeSnapshot = meta?.isHedgeWinner ? await meta.hedgeBindingSnapshotPromise : null; + if (hedgeSnapshot) { + const cleared = await SessionManager.clearVersionedSessionProvider( + hedgeSnapshot, + providerIdForPersistence + ); + if (cleared.status !== "ok") { + logger.warn("[ResponseHandler] Hedge winner binding clear stopped", { + sessionId: hedgeSnapshot.sessionId, + providerId: providerIdForPersistence, + reason: cleared.reason, + }); + } + return; + } const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; await SessionManager.clearSessionProvider(session.sessionId, providerIdForPersistence, keyId); }; @@ -1240,11 +1346,15 @@ function finalizeDeferredStreamingFinalizationIfNeeded( ((clientAborted || !streamEndedNormally) && !clientAbortCompleteSuccess) || detected.isError || (upstreamStatusCode >= 400 && errorMessage !== null); + if (shouldClearSessionBindingOnFailure) { + meta?.hedgeBindingHeartbeat?.stop(); + } // 未启用延迟结算 / provider 缺失： // - 只返回“内部状态码 + 错误原因”，由调用方写入统计； // - 不在这里更新熔断/绑定（meta 缺失意味着 Forwarder 没有启用延迟结算；provider 缺失意味着无法归因）。 if (!meta || !provider) { + meta?.hedgeBindingHeartbeat?.stop(); return { effectiveStatusCode, errorMessage, @@ -1462,7 +1572,13 @@ function finalizeDeferredStreamingFinalizationIfNeeded( }); } + // Stop periodic refresh at the stream boundary and issue one final + // generation-safe touch so the next turn receives a full binding TTL. + // complete() is idempotent and permanently stops after any authority conflict. + const hedgeBindingCompletion = meta.hedgeBindingHeartbeat?.complete(); const commitSideEffects = async () => { + await hedgeBindingCompletion; + if (meta.endpointId != null) { try { const { recordEndpointSuccess } = await import("@/lib/endpoint-circuit-breaker"); @@ -2536,6 +2652,8 @@ export class ProxyResponseHandler { return response; } + startHedgeBindingHeartbeat(session); + let processedStream: ReadableStream = response.body; // --- GEMINI STREAM HANDLING --- diff --git a/src/app/v1/_lib/proxy/stream-finalization.ts b/src/app/v1/_lib/proxy/stream-finalization.ts index 0f989bb3b..9062ef11c 100644 --- a/src/app/v1/_lib/proxy/stream-finalization.ts +++ b/src/app/v1/_lib/proxy/stream-finalization.ts @@ -1,5 +1,11 @@ +import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import type { ProxySession } from "./session"; +export type DeferredStreamingBindingHeartbeat = { + stop: () => void; + complete: () => Promise; +}; + /** * 流式响应（SSE）在“收到响应头”时无法确定成功与否： * - 上游可能返回 HTTP 200，但 body 是错误 JSON（假 200） @@ -35,6 +41,10 @@ export type DeferredStreamingFinalization = { * coexists with asynchronously accumulated loser costs without clobbering. */ billHedgeLosers?: boolean; + /** Exact generation created by the legacy Hedge winner's first-byte CAS. */ + hedgeBindingSnapshotPromise?: Promise; + /** ResponseHandler-owned runtime lifecycle; attached when streaming starts. */ + hedgeBindingHeartbeat?: DeferredStreamingBindingHeartbeat; }; const deferredMeta = new WeakMap(); diff --git a/src/lib/redis/session-binding.ts b/src/lib/redis/session-binding.ts index 327979d9d..a72ebb322 100644 --- a/src/lib/redis/session-binding.ts +++ b/src/lib/redis/session-binding.ts @@ -1463,12 +1463,34 @@ export async function mutateLegacySessionBindingSafely( }; } - if (legacyProvider !== null) await redis.del(keys.legacyProvider); - await redis.del(keys.legacyOwner); + if (legacyProvider !== null) { + const deleted = await deleteLegacyProviderIfValue( + redis, + keys.legacyProvider, + legacyProvider + ); + if (!deleted) { + const conflictAfterConcurrentMutation = + await rejectLegacyMutationAfterCanonicalAppeared(redis, keys); + if (conflictAfterConcurrentMutation) return conflictAfterConcurrentMutation; + return conflict("provider_mismatch"); + } + } + + // Keep the tenant owner as a null-binding tombstone. Besides matching + // the versioned terminate semantics, this prevents a recovered + // versioned worker from losing its required owner mirror while this + // fallback operation is in flight. + { + const ownerConflict = await ensureLegacyOwner(redis, keys, input.keyId, ttlSeconds); + if (ownerConflict) return ownerConflict; + } { const conflictAfterTerminate = await rejectLegacyMutationAfterCanonicalAppeared( redis, - keys + keys, + undefined, + legacyProvider === null ? undefined : { value: legacyProvider.toString(), ttlSeconds } ); if (conflictAfterTerminate) return conflictAfterTerminate; } diff --git a/src/lib/session-manager.ts b/src/lib/session-manager.ts index 341b407a3..1a0e3450a 100644 --- a/src/lib/session-manager.ts +++ b/src/lib/session-manager.ts @@ -1125,7 +1125,12 @@ export class SessionManager { isFailoverSuccess: boolean = false, keyId?: number | null, forceUpdate: boolean = false - ): Promise<{ updated: boolean; reason: string; details?: string }> { + ): Promise<{ + updated: boolean; + reason: string; + details?: string; + bindingSnapshot?: SessionBindingSnapshot; + }> { const redis = getRedisClient(); if (!redis || redis.status !== "ready") { return { updated: false, reason: "redis_not_ready" }; @@ -1133,6 +1138,7 @@ export class SessionManager { try { let versionedSnapshot: SessionBindingSnapshot | null = null; + let committedVersionedSnapshot: SessionBindingSnapshot | null = null; let useLegacyBinding = false; let legacyProviderId: number | null = null; @@ -1199,6 +1205,7 @@ export class SessionManager { }); return false; } + committedVersionedSnapshot = result.snapshot; return true; } @@ -1263,6 +1270,7 @@ export class SessionManager { details: isFailoverSuccess ? `故障转移成功，绑定到供应商 ${newProviderId}` : `竞速赢家强制改绑到供应商 ${newProviderId}`, + ...(committedVersionedSnapshot ? { bindingSnapshot: committedVersionedSnapshot } : {}), }; } diff --git a/tests/unit/lib/redis/session-binding.test.ts b/tests/unit/lib/redis/session-binding.test.ts index a3132ada5..2831116b9 100644 --- a/tests/unit/lib/redis/session-binding.test.ts +++ b/tests/unit/lib/redis/session-binding.test.ts @@ -1370,6 +1370,90 @@ describe("versioned session binding operations", () => { expect(mock.delMock).not.toHaveBeenCalled(); }); + it("keeps the tenant owner tombstone after an unscoped legacy termination", async () => { + let owner: string | null = "4"; + let provider: string | null = "8"; + const mock = createMockRedis(); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return owner; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.delMock.mockImplementation(async (key: string) => { + if (key === "session:sid:provider") provider = null; + if (key === "session:sid:key") owner = null; + return 1; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "terminate" }, + }); + + expect(result).toMatchObject({ status: "ok", changed: true, providerId: null }); + expect(provider).toBeNull(); + expect(owner).toBe("4"); + expect(mock.delMock).not.toHaveBeenCalledWith("session:sid:key"); + }); + + it("restores mirrors when versioned recovery races an unscoped legacy termination", async () => { + let owner: string | null = "4"; + let provider: string | null = "8"; + const mock = createMockRedis({ + operationResponses: { + [RESTORE_LEGACY_PROVIDER_IF_ABSENT]: [ + () => { + if (provider === null) provider = "8"; + return 1; + }, + ], + }, + }); + let existsCalls = 0; + mock.existsMock.mockImplementation(async () => { + existsCalls += 1; + return existsCalls === 3 ? 1 : 0; + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return owner; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.hgetMock.mockResolvedValue("8"); + mock.delMock.mockImplementation(async (key: string) => { + if (key === "session:sid:provider") provider = null; + if (key === "session:sid:key") owner = null; + return 1; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "terminate" }, + }); + + expect(result).toMatchObject({ status: "conflict", reason: "canonical_exists" }); + expect(owner).toBe("4"); + expect(provider).toBe("8"); + expect(mock.evalMock).toHaveBeenCalledWith( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + "session:sid:provider", + "8" + ); + expect(mock.evalMock).toHaveBeenCalledWith( + RESTORE_LEGACY_PROVIDER_IF_ABSENT, + 1, + "session:sid:provider", + "8", + "300" + ); + expect(mock.delMock).not.toHaveBeenCalledWith("session:sid:key"); + }); + it("uses the tenant-authorized termination primitive and leaves a tombstone", async () => { const mock = createMockRedis({ operationResponses: { diff --git a/tests/unit/lib/session-manager-binding-smart.test.ts b/tests/unit/lib/session-manager-binding-smart.test.ts index b7d9600b6..81d91e662 100644 --- a/tests/unit/lib/session-manager-binding-smart.test.ts +++ b/tests/unit/lib/session-manager-binding-smart.test.ts @@ -278,6 +278,12 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { providerId: 2, }) ); + expect(result.bindingSnapshot).toEqual({ + sessionId: SID, + keyId: 42, + providerId: 2, + generation: "generation-b", + }); expect(redisClientRef!.pipeline).not.toHaveBeenCalled(); expect(redisClientRef!.setex).not.toHaveBeenCalled(); }); diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index fe5d7f4a8..fabe26ffd 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -123,6 +123,7 @@ import { import { ProxyForwarder } from "@/app/v1/_lib/proxy/forwarder"; import { ModelRedirector } from "@/app/v1/_lib/proxy/model-redirector"; import { ProxySession } from "@/app/v1/_lib/proxy/session"; +import { peekDeferredStreamingFinalization } from "@/app/v1/_lib/proxy/stream-finalization"; import { logger } from "@/lib/logger"; import type { Provider } from "@/types/provider"; @@ -922,7 +923,22 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100 }); const session = createSession(); + session.authState = { + ...session.authState!, + key: { id: 456 }, + } as never; setProviderWithSessionRef(session, provider1); + const winnerBindingSnapshot = { + sessionId: "sess-hedge", + keyId: 456, + providerId: 2, + generation: "hedge-winner-generation", + }; + mocks.updateSessionBindingSmart.mockResolvedValueOnce({ + updated: true, + reason: "race_winner_forced", + bindingSnapshot: winnerBindingSnapshot, + } as never); mocks.pickRandomProviderWithExclusion.mockResolvedValueOnce(provider2); @@ -965,7 +981,9 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { await vi.advanceTimersByTimeAsync(50); const response = await responsePromise; + const deferred = peekDeferredStreamingFinalization(session); expect(await response.text()).toContain('"provider":"p2"'); + await expect(deferred?.hedgeBindingSnapshotPromise).resolves.toEqual(winnerBindingSnapshot); expect(controller1.signal.aborted).toBe(true); expect(controller2.signal.aborted).toBe(false); expect(mocks.recordFailure).not.toHaveBeenCalled(); @@ -979,7 +997,7 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { 0, false, true, - null, + 456, true ); expect(mocks.releaseProviderSession).toHaveBeenCalledWith(1, "sess-hedge"); diff --git a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts index af432f126..02cc32cc4 100644 --- a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts +++ b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts @@ -74,10 +74,13 @@ vi.mock("@/repository/message", () => ({ vi.mock("@/lib/session-manager", () => ({ SessionManager: { + clearVersionedSessionProvider: vi.fn(), updateSessionUsage: vi.fn(), storeSessionResponse: vi.fn(), clearSessionProvider: vi.fn(), extractCodexPromptCacheKey: vi.fn(), + getVersionedSessionBindingRefreshIntervalMs: vi.fn(), + touchVersionedSessionBinding: vi.fn(), updateSessionBindingSmart: vi.fn(), updateSessionProvider: vi.fn(), updateSessionWithCodexCacheKey: vi.fn(), @@ -272,7 +275,11 @@ function createSession(opts?: { sessionId?: string | null }): ProxySession { return session; } -function setDeferredMeta(session: ProxySession, endpointId: number | null = 42) { +function setDeferredMeta( + session: ProxySession, + endpointId: number | null = 42, + extra: Partial[1]> = {} +) { setDeferredStreamingFinalization(session, { providerId: 1, providerName: "test-provider", @@ -284,6 +291,7 @@ function setDeferredMeta(session: ProxySession, endpointId: number | null = 42) endpointId, endpointUrl: "https://api.test.com", upstreamStatusCode: 200, + ...extra, }); } @@ -335,6 +343,31 @@ function createSuccessStreamResponse(): Response { }); } +function createControllableSuccessStreamResponse(): { + response: Response; + complete: () => void; +} { + const encoder = new TextEncoder(); + let sourceController: ReadableStreamDefaultController | null = null; + const stream = new ReadableStream({ + start(controller) { + sourceController = controller; + controller.enqueue( + encoder.encode( + `event: message_delta\ndata: ${JSON.stringify({ usage: { input_tokens: 100, output_tokens: 50 } })}\n\n` + ) + ); + }, + }); + return { + response: new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), + complete: () => sourceController?.close(), + }; +} + async function drainAsyncTasks(): Promise { while (asyncTasks.length > 0) { const tasks = asyncTasks.splice(0, asyncTasks.length); @@ -366,6 +399,24 @@ function setupCommonMocks() { vi.mocked(updateMessageRequestDuration).mockResolvedValue(undefined); vi.mocked(SessionManager.storeSessionResponse).mockResolvedValue(undefined); vi.mocked(SessionManager.clearSessionProvider).mockResolvedValue(undefined); + vi.mocked(SessionManager.clearVersionedSessionProvider).mockResolvedValue({ + status: "ok", + source: "cleared", + snapshot: { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "cleared-generation", + }, + legacyFallbackAllowed: false, + }); + vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(100_000); + vi.mocked(SessionManager.touchVersionedSessionBinding).mockImplementation(async (snapshot) => ({ + status: "ok", + source: "touched", + snapshot, + legacyFallbackAllowed: false, + })); vi.mocked(SessionManager.updateSessionUsage).mockResolvedValue(undefined); vi.mocked(SessionManager.updateSessionBindingSmart).mockResolvedValue({ updated: true, @@ -537,4 +588,132 @@ describe("Endpoint circuit breaker isolation", () => { expect(mockRecordEndpointSuccess).not.toHaveBeenCalled(); expect(mockRecordEndpointFailure).not.toHaveBeenCalled(); }); + + it("keeps the captured legacy Hedge winner binding alive throughout a long stream", async () => { + vi.useFakeTimers(); + try { + const session = createSession(); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: 1, + generation: "hedge-long-stream-generation", + } as const; + setDeferredMeta(session, 42, { + isHedgeWinner: true, + hedgeBindingSnapshotPromise: Promise.resolve(snapshot), + }); + vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(1_000); + const controlled = createControllableSuccessStreamResponse(); + + const clientResponse = await ProxyResponseHandler.dispatch(session, controlled.response); + const bodyPromise = clientResponse.text(); + await vi.advanceTimersByTimeAsync(3_000); + + // Immediate ownership validation plus one heartbeat per interval. + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes(4); + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenLastCalledWith(snapshot); + + controlled.complete(); + await bodyPromise; + await drainAsyncTasks(); + + // A final touch gives the next turn a complete TTL from stream completion. + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes(5); + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenLastCalledWith(snapshot); + expect(SessionManager.updateSessionBindingSmart).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(3_000); + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes(5); + } finally { + vi.useRealTimers(); + } + }); + + it("clears a failed Hedge stream only through its captured generation", async () => { + vi.useFakeTimers(); + try { + const session = createSession(); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: 1, + generation: "failed-hedge-generation", + } as const; + setDeferredMeta(session, 42, { + isHedgeWinner: true, + hedgeBindingSnapshotPromise: Promise.resolve(snapshot), + }); + vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(1_000); + vi.mocked(SessionManager.clearVersionedSessionProvider).mockResolvedValueOnce({ + status: "conflict", + reason: "generation_mismatch", + legacyFallbackAllowed: false, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createFake200StreamResponse() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.clearVersionedSessionProvider).toHaveBeenCalledWith(snapshot, 1); + expect(SessionManager.clearSessionProvider).not.toHaveBeenCalled(); + const touchesAfterFailure = vi.mocked(SessionManager.touchVersionedSessionBinding).mock.calls + .length; + await vi.advanceTimersByTimeAsync(5_000); + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes( + touchesAfterFailure + ); + } finally { + vi.useRealTimers(); + } + }); + + it("stops Hedge heartbeats after a generation conflict and never revives the binding", async () => { + vi.useFakeTimers(); + try { + const session = createSession(); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: 1, + generation: "generation-before-termination", + } as const; + setDeferredMeta(session, 42, { + isHedgeWinner: true, + hedgeBindingSnapshotPromise: Promise.resolve(snapshot), + }); + vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(1_000); + vi.mocked(SessionManager.touchVersionedSessionBinding) + .mockResolvedValueOnce({ + status: "ok", + source: "touched", + snapshot, + legacyFallbackAllowed: false, + }) + .mockResolvedValueOnce({ + status: "conflict", + reason: "generation_mismatch", + legacyFallbackAllowed: false, + }); + const controlled = createControllableSuccessStreamResponse(); + + const clientResponse = await ProxyResponseHandler.dispatch(session, controlled.response); + const bodyPromise = clientResponse.text(); + await vi.advanceTimersByTimeAsync(5_000); + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes(2); + + controlled.complete(); + await bodyPromise; + await drainAsyncTasks(); + + // Completion must not retry after an administrator or concurrent request + // advances the generation. + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes(2); + expect(SessionManager.updateSessionBindingSmart).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); }); From 811192a10baeb3a0d1b7f3d6a07bf95c2788fecc Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 20:57:19 -0400 Subject: [PATCH 52/89] fix(discovery): reject failed response terminal markers --- src/app/v1/_lib/proxy/discovery-validity.ts | 26 ++++++++- src/app/v1/_lib/proxy/response-handler.ts | 2 + tests/unit/proxy/discovery-validity.test.ts | 15 ++++++ ...handler-endpoint-circuit-isolation.test.ts | 53 +++++++++++++++++++ 4 files changed, 94 insertions(+), 2 deletions(-) diff --git a/src/app/v1/_lib/proxy/discovery-validity.ts b/src/app/v1/_lib/proxy/discovery-validity.ts index 059bde293..81a16885d 100644 --- a/src/app/v1/_lib/proxy/discovery-validity.ts +++ b/src/app/v1/_lib/proxy/discovery-validity.ts @@ -49,15 +49,37 @@ function hasAnthropicContentBlock(value: unknown): boolean { return block.type === "text" ? hasContent(block.text) : true; } -function classifyJson(value: unknown, protocol: DiscoveryProtocol): DiscoveryValidity { - if (!value || typeof value !== "object") return { ready: false, terminal: false, error: true }; +/** + * Protocol-level error signals that must remain terminal even if a provider + * emits a later completion marker. Keep this shared by the racing parser and + * stream finalizer so a failed winner cannot become Sticky during settlement. + */ +export function isDiscoveryProtocolErrorPayload(value: unknown): boolean { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; const object = value as Record; if ( object.error || object.failed || object.type === "error" || + object.type === "response.error" || object.type === "response.failed" ) { + return true; + } + + const response = object.response; + return ( + !!response && + typeof response === "object" && + !Array.isArray(response) && + !!(response as Record).error + ); +} + +function classifyJson(value: unknown, protocol: DiscoveryProtocol): DiscoveryValidity { + if (!value || typeof value !== "object") return { ready: false, terminal: false, error: true }; + const object = value as Record; + if (isDiscoveryProtocolErrorPayload(value)) { return { ready: false, terminal: true, error: true }; } if (protocol === "openai-chat") { diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index aa66e78c2..14b532cef 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -56,6 +56,7 @@ import { createDemandDrivenResponsePump, type DemandDrivenResponsePump, } from "./demand-driven-response-pump"; +import { isDiscoveryProtocolErrorPayload } from "./discovery-validity"; import { isClientAbortError, isTransportError } from "./errors"; import type { ProxySession } from "./session"; import { @@ -1197,6 +1198,7 @@ function hasStreamCompletionMarker(text: string, format: ProxySession["originalF switch (format) { case "response": + if (events.some((event) => isDiscoveryProtocolErrorPayload(event.data))) return false; return events.some((event) => { if (!isRecord(event.data)) return false; const markerType = event.data.type; diff --git a/tests/unit/proxy/discovery-validity.test.ts b/tests/unit/proxy/discovery-validity.test.ts index 5093298dd..46ee321c2 100644 --- a/tests/unit/proxy/discovery-validity.test.ts +++ b/tests/unit/proxy/discovery-validity.test.ts @@ -33,6 +33,21 @@ describe("discovery validity", () => { expect(parser.push('{"type":"response.output_text.delta","delta":"late"}').ready).toBe(false); }); + it.each([ + '{"type":"response.error"}', + '{"failed":true}', + '{"type":"response.done","response":{"error":{"message":"no"}}}', + ])("rejects Responses protocol error payload %s", (payload) => { + const parser = new DiscoveryValidityParser("openai-responses"); + + expect(parser.push(payload)).toMatchObject({ ready: false, terminal: true, error: true }); + expect(parser.push('{"type":"response.done"}')).toMatchObject({ + ready: false, + terminal: true, + error: true, + }); + }); + it("does not promote empty tool or content events", () => { expect( classifyDiscoveryChunk( diff --git a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts index e076c2d8f..5ddb7a2f9 100644 --- a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts +++ b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts @@ -853,6 +853,59 @@ describe("Endpoint circuit breaker isolation", () => { ); }); + it("does not let response.done override an earlier nested Responses failure", async () => { + const session = createSession(); + session.originalFormat = "response"; + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "failed-before-done-generation", + } as const; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: snapshot, + requiresCompletionMarker: true, + }); + const body = + `event: response.output_text.delta\ndata: ${JSON.stringify({ + type: "response.output_text.delta", + delta: "partial output", + })}\n\n` + + `event: response.failed\ndata: ${JSON.stringify({ + type: "response.failed", + response: { status: "failed", error: { message: "upstream failed" } }, + })}\n\n` + + `event: response.done\ndata: ${JSON.stringify({ type: "response.done" })}\n\n`; + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }) + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); + expect(mockRecordSuccess).not.toHaveBeenCalled(); + expect(mockRecordFailure).toHaveBeenCalledWith( + 1, + expect.objectContaining({ message: "STREAM_COMPLETION_MARKER_MISSING" }) + ); + }); + it.each([ { label: "Claude data-only stop", From d915e086596d9594da110f9b2476294b52635edc Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 21:01:41 -0400 Subject: [PATCH 53/89] fix(discovery): refill setup failures within current round --- src/app/v1/_lib/proxy/forwarder.ts | 146 +++++++++--------- .../proxy-forwarder-hedge-first-byte.test.ts | 82 ++++++++++ 2 files changed, 152 insertions(+), 76 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index ffaf62d68..3e88aaee0 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -5284,19 +5284,6 @@ export class ProxyForwarder { }); }; - const chooseCandidate = async (): Promise => { - const candidates = await ProxyProviderResolver.pickDiscoveryProviders( - session, - 1, - Array.from(launched) - ); - if (!candidates[0]) { - noMoreCandidates = true; - return null; - } - return candidates[0]; - }; - const clearCapturedStickyBinding = async (cooldownTtlSeconds: number): Promise => { if ( !bindingWriteAllowed || @@ -5792,19 +5779,7 @@ export class ProxyForwarder { } } if (!actionOwnsNextStep && !committed && !settled) { - const replacement = await chooseCandidate(); - if (replacement) { - try { - await launch(replacement, "normal"); - } catch (launchError) { - lastError = - launchError instanceof Error ? launchError : new Error(String(launchError)); - // A single launch failure does not prove that the remaining - // candidate pool is exhausted. The current round boundary can - // still advance or retry selection from the remaining pool. - noMoreCandidates = false; - } - } + await refillCurrentRoundSlots(1); } if ( Array.from(attempts.values()).every((candidate) => !candidate.pending) && @@ -5824,6 +5799,73 @@ export class ProxyForwarder { return true; }; + const fillDiscoverySlots = async (slots: number): Promise => { + let remainingSlots = slots; + while (remainingSlots > 0 && !settled && !committed) { + const exclusionCountBeforeSelection = launched.size; + const candidates = await ProxyProviderResolver.pickDiscoveryProviders( + session, + remainingSlots, + Array.from(launched) + ); + if (candidates.length === 0) { + noMoreCandidates = true; + break; + } + + noMoreCandidates = false; + let registeredInBatch = 0; + for (const candidate of candidates) { + try { + if (await launch(candidate, "normal")) { + registeredInBatch += 1; + remainingSlots -= 1; + } + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + // launch() excludes setup failures before throwing, so keep filling + // this round from the remaining candidate pool. + noMoreCandidates = false; + } + if (remainingSlots <= 0 || settled || committed) break; + } + + // A selector that ignores exclusions must not create an unbounded setup + // loop. Normal selectors advance `launched` even before transport setup. + if (registeredInBatch === 0 && launched.size === exclusionCountBeforeSelection) { + noMoreCandidates = true; + break; + } + } + }; + + const finishRoundLaunchBatch = async (): Promise => { + roundLaunchesInProgress = Math.max(0, roundLaunchesInProgress - 1); + if (roundLaunchesInProgress !== 0) return; + + notifyRoundLaunchIdle(); + fallbackPromotionBlocked = false; + const readyFallback = Array.from(attempts.values()).find( + (attempt) => attempt.pending && attempt.ready && attempt.kind === "fallback" + ); + if (readyFallback && !committed && !settled) { + const action = coordinator.markReady(readyFallback.id); + if (action.type === "promote_fallback") await commit(readyFallback); + } + }; + + async function refillCurrentRoundSlots(slots: number): Promise { + if (slots <= 0 || settled || committed) return; + roundLaunchesInProgress += 1; + try { + // Deliberately preserve the existing round timer. An explicit failure + // releases capacity but must not grant the replacement a fresh SLA. + await fillDiscoverySlots(slots); + } finally { + await finishRoundLaunchBatch(); + } + } + const launchNextRound = async (slots: number, coordinatorAlreadyAdvanced = false) => { if (settled || committed) return; roundLaunchesInProgress += 1; @@ -5836,44 +5878,7 @@ export class ProxyForwarder { currentRound = nextRound.round; } if (currentRound > maxRounds || slots <= 0) return; - let remainingSlots = slots; - while (remainingSlots > 0 && !settled && !committed) { - const exclusionCountBeforeSelection = launched.size; - const candidates = await ProxyProviderResolver.pickDiscoveryProviders( - session, - remainingSlots, - Array.from(launched) - ); - if (candidates.length === 0) { - noMoreCandidates = true; - break; - } - - noMoreCandidates = false; - let registeredInBatch = 0; - for (const candidate of candidates) { - try { - if (await launch(candidate, "normal")) { - registeredInBatch += 1; - remainingSlots -= 1; - } - } catch (error) { - lastError = error instanceof Error ? error : new Error(String(error)); - // This Provider is already excluded by launch(). Continue filling - // the same round from the remaining candidate pool. - noMoreCandidates = false; - } - if (remainingSlots <= 0 || settled || committed) break; - } - - // A selector that ignores exclusions must not create an unbounded - // setup loop. Normal selectors always advance `launched`, including - // attempts that fail before transport registration. - if (registeredInBatch === 0 && launched.size === exclusionCountBeforeSelection) { - noMoreCandidates = true; - break; - } - } + await fillDiscoverySlots(slots); const hasPendingAttempt = Array.from(attempts.values()).some((attempt) => attempt.pending); if (!hasPendingAttempt) { await settleFailure(ProxyForwarder.buildAllProvidersUnavailableError(lastError)); @@ -5883,18 +5888,7 @@ export class ProxyForwarder { scheduleRoundBoundary(discoverySlaMs); } } finally { - roundLaunchesInProgress = Math.max(0, roundLaunchesInProgress - 1); - if (roundLaunchesInProgress === 0) { - notifyRoundLaunchIdle(); - fallbackPromotionBlocked = false; - const readyFallback = Array.from(attempts.values()).find( - (attempt) => attempt.pending && attempt.ready && attempt.kind === "fallback" - ); - if (readyFallback && !committed && !settled) { - const action = coordinator.markReady(readyFallback.id); - if (action.type === "promote_fallback") await commit(readyFallback); - } - } + await finishRoundLaunchBatch(); } }; diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index bec72c795..9d8cd16ab 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -2943,6 +2943,88 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { } }); + test("Discovery keeps refilling the current round when an error replacement fails setup", async () => { + const initialFailure = createProvider({ id: 1, name: "initial-failure", priority: 1 }); + const pending = createProvider({ id: 2, name: "pending", priority: 1 }); + const setupFailure = createProvider({ id: 3, name: "setup-failure", priority: 1 }); + const healthy = createProvider({ id: 4, name: "healthy", priority: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 31 }, + apiKey: null, + } as typeof session.authState; + session.setProvider(initialFailure); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 50, + stickySlaMs: 50, + racingTotalTimeoutMs: 200, + stickyTimeoutCooldownMs: 300_000, + }); + mocks.pickDiscoveryProviders + .mockResolvedValueOnce([pending]) + .mockResolvedValueOnce([setupFailure]) + .mockResolvedValueOnce([healthy]); + + const endpointResolver = vi.spyOn( + ProxyForwarder as unknown as { + resolveStreamingHedgeEndpoint: ( + session: ProxySession, + provider: Provider + ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + }, + "resolveStreamingHedgeEndpoint" + ); + endpointResolver.mockImplementation(async (_attemptSession, provider) => { + if (provider.id === setupFailure.id) throw new Error("replacement setup failed"); + return { + endpointId: null, + baseUrl: provider.url, + endpointUrl: provider.url, + }; + }); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockImplementation(async (attemptSession) => { + const providerId = (attemptSession as ProxySession).provider?.id; + if (providerId === initialFailure.id) throw new Error("initial Provider failed"); + if (providerId === healthy.id) { + return new Response('data: {"type":"content_block_delta","delta":{"text":"healthy"}}\n\n', { + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response(new ReadableStream(), { + headers: { "content-type": "text/event-stream" }, + }); + }); + + try { + const response = await ProxyForwarder.send(session); + + expect(await response.text()).toContain('"healthy"'); + expect(mocks.pickDiscoveryProviders).toHaveBeenCalledTimes(3); + expect(mocks.pickDiscoveryProviders).toHaveBeenNthCalledWith( + 3, + expect.anything(), + 1, + expect.arrayContaining([initialFailure.id, pending.id, setupFailure.id]) + ); + expect(doForward).toHaveBeenCalledTimes(3); + expect(session.provider?.id).toBe(healthy.id); + } finally { + endpointResolver.mockRestore(); + } + }); + test("an explicit Sticky failure starts Discovery round one at full concurrency", async () => { const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); const normal = createProvider({ id: 2, name: "normal", priority: 1 }); From 81d9b4f446b67e4c3def01359b4f17967386542c Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 21:10:46 -0400 Subject: [PATCH 54/89] fix(binding): fence failed hedge cleanup --- src/app/v1/_lib/proxy/forwarder.ts | 26 +++++--- src/app/v1/_lib/proxy/response-handler.ts | 16 +++-- src/app/v1/_lib/proxy/stream-finalization.ts | 11 +++- src/lib/session-manager.ts | 7 ++- .../lib/session-manager-binding-smart.test.ts | 22 +++++++ .../proxy-forwarder-hedge-first-byte.test.ts | 5 +- ...handler-endpoint-circuit-isolation.test.ts | 61 ++++++++++++++++++- 7 files changed, 127 insertions(+), 21 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 042aa9f0c..e5e8297a2 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -30,7 +30,6 @@ import { } from "@/lib/provider-endpoints/endpoint-selector"; import { getGlobalAgentPool, getProxyAgentForProvider } from "@/lib/proxy-agent"; import { RateLimitService } from "@/lib/rate-limit/service"; -import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import { SessionManager } from "@/lib/session-manager"; import { detectUpstreamErrorFromSseOrJsonText, @@ -99,7 +98,10 @@ import { import { ProxyProviderResolver } from "./provider-selector"; import { finalizeHedgeLoserBilling } from "./response-handler"; import type { ProxySession } from "./session"; -import { setDeferredStreamingFinalization } from "./stream-finalization"; +import { + type DeferredStreamingHedgeBindingAuthority, + setDeferredStreamingFinalization, +} from "./stream-finalization"; import { detectThinkingBudgetRectifierTrigger, rectifyThinkingBudget, @@ -4558,9 +4560,9 @@ export class ProxyForwarder { // A non-hedged request is finalized through response-handler. Updating // here as well would perform a duplicate binding read/CAS before the // stream has passed its final validation. - let hedgeBindingSnapshotPromise: Promise | undefined; + let hedgeBindingAuthorityPromise: Promise | undefined; if (session.sessionId && isActualHedgeWin) { - hedgeBindingSnapshotPromise = (async () => { + hedgeBindingAuthorityPromise = (async () => { const bindingResult = await SessionManager.updateSessionBindingSmart( session.sessionId!, attempt.provider.id, @@ -4594,14 +4596,20 @@ export class ProxyForwarder { }); } - // Only the exact snapshot returned by the first-byte CAS may keep - // this binding alive. Legacy fallback and failed CAS paths return null. - return bindingResult.bindingSnapshot ?? null; + return { + // Only the exact snapshot returned by the first-byte CAS may keep + // a versioned binding alive or clear it after a failed stream. + snapshot: bindingResult.bindingSnapshot ?? null, + // A generic clear is safe only when this request demonstrably + // committed the legacy binding. A versioned CAS conflict must not + // clear a newer generation that happens to use the same Provider. + legacyClearAllowed: bindingResult.legacyBindingUpdated === true, + }; })().catch((bindingError) => { logger.error("ProxyForwarder: Failed to update session provider info for hedge winner", { error: bindingError, }); - return null; + return { snapshot: null, legacyClearAllowed: false }; }); } @@ -4618,7 +4626,7 @@ export class ProxyForwarder { upstreamStatusCode: attempt.response.status, isHedgeWinner: isActualHedgeWin, billHedgeLosers, - hedgeBindingSnapshotPromise, + hedgeBindingAuthorityPromise, }); const response = new Response( diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index c87fd6ad9..d5a522a66 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -100,8 +100,8 @@ const NON_STREAM_TERMINAL_PERSISTENCE_ERROR = Symbol("non_stream_terminal_persis function startHedgeBindingHeartbeat(session: ProxySession): void { const deferred = peekDeferredStreamingFinalization(session); - const snapshotPromise = deferred?.hedgeBindingSnapshotPromise; - if (!deferred?.isHedgeWinner || !snapshotPromise || deferred.hedgeBindingHeartbeat) return; + const authorityPromise = deferred?.hedgeBindingAuthorityPromise; + if (!deferred?.isHedgeWinner || !authorityPromise || deferred.hedgeBindingHeartbeat) return; let periodicActive = true; let authorityLost = false; @@ -133,7 +133,7 @@ function startHedgeBindingHeartbeat(session: ProxySession): void { if (touchInFlight) return touchInFlight; const operation = (async () => { - const snapshot = await snapshotPromise; + const { snapshot } = await authorityPromise; if (!snapshot) { authorityLost = true; stopPeriodic(); @@ -1240,8 +1240,11 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const providerIdForPersistence = meta?.providerId ?? provider?.id ?? null; const clearSessionBinding = async () => { if (!session.sessionId) return; - const hedgeSnapshot = meta?.isHedgeWinner ? await meta.hedgeBindingSnapshotPromise : null; - if (hedgeSnapshot) { + const hedgeAuthority = meta?.isHedgeWinner + ? await meta.hedgeBindingAuthorityPromise + : undefined; + if (hedgeAuthority?.snapshot) { + const hedgeSnapshot = hedgeAuthority.snapshot; const cleared = await SessionManager.clearVersionedSessionProvider( hedgeSnapshot, providerIdForPersistence @@ -1255,6 +1258,9 @@ function finalizeDeferredStreamingFinalizationIfNeeded( } return; } + if (meta?.isHedgeWinner && !hedgeAuthority?.legacyClearAllowed) { + return; + } const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; await SessionManager.clearSessionProvider(session.sessionId, providerIdForPersistence, keyId); }; diff --git a/src/app/v1/_lib/proxy/stream-finalization.ts b/src/app/v1/_lib/proxy/stream-finalization.ts index 9062ef11c..c1eafe42b 100644 --- a/src/app/v1/_lib/proxy/stream-finalization.ts +++ b/src/app/v1/_lib/proxy/stream-finalization.ts @@ -6,6 +6,13 @@ export type DeferredStreamingBindingHeartbeat = { complete: () => Promise; }; +export type DeferredStreamingHedgeBindingAuthority = { + /** Exact versioned generation written by the first-byte winner, when available. */ + snapshot: SessionBindingSnapshot | null; + /** Only a confirmed successful legacy write may use the non-versioned clear path. */ + legacyClearAllowed: boolean; +}; + /** * 流式响应（SSE）在“收到响应头”时无法确定成功与否： * - 上游可能返回 HTTP 200，但 body 是错误 JSON（假 200） @@ -41,8 +48,8 @@ export type DeferredStreamingFinalization = { * coexists with asynchronously accumulated loser costs without clobbering. */ billHedgeLosers?: boolean; - /** Exact generation created by the legacy Hedge winner's first-byte CAS. */ - hedgeBindingSnapshotPromise?: Promise; + /** Binding authority established by the legacy Hedge winner's first-byte write. */ + hedgeBindingAuthorityPromise?: Promise; /** ResponseHandler-owned runtime lifecycle; attached when streaming starts. */ hedgeBindingHeartbeat?: DeferredStreamingBindingHeartbeat; }; diff --git a/src/lib/session-manager.ts b/src/lib/session-manager.ts index 1a0e3450a..cd829b620 100644 --- a/src/lib/session-manager.ts +++ b/src/lib/session-manager.ts @@ -1130,6 +1130,7 @@ export class SessionManager { reason: string; details?: string; bindingSnapshot?: SessionBindingSnapshot; + legacyBindingUpdated?: boolean; }> { const redis = getRedisClient(); if (!redis || redis.status !== "ready") { @@ -1139,6 +1140,7 @@ export class SessionManager { try { let versionedSnapshot: SessionBindingSnapshot | null = null; let committedVersionedSnapshot: SessionBindingSnapshot | null = null; + let committedLegacyBinding = false; let useLegacyBinding = false; let legacyProviderId: number | null = null; @@ -1219,7 +1221,9 @@ export class SessionManager { ? { type: "bind_if_absent", providerId: newProviderId } : { type: "set", providerId: newProviderId }, }); - return result.status === "ok" && result.changed; + const updated = result.status === "ok" && result.changed; + if (updated) committedLegacyBinding = true; + return updated; }; if (isFirstAttempt) { @@ -1271,6 +1275,7 @@ export class SessionManager { ? `故障转移成功，绑定到供应商 ${newProviderId}` : `竞速赢家强制改绑到供应商 ${newProviderId}`, ...(committedVersionedSnapshot ? { bindingSnapshot: committedVersionedSnapshot } : {}), + ...(committedLegacyBinding ? { legacyBindingUpdated: true } : {}), }; } diff --git a/tests/unit/lib/session-manager-binding-smart.test.ts b/tests/unit/lib/session-manager-binding-smart.test.ts index 81d91e662..62c188797 100644 --- a/tests/unit/lib/session-manager-binding-smart.test.ts +++ b/tests/unit/lib/session-manager-binding-smart.test.ts @@ -284,10 +284,32 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { providerId: 2, generation: "generation-b", }); + expect(result.legacyBindingUpdated).toBeUndefined(); expect(redisClientRef!.pipeline).not.toHaveBeenCalled(); expect(redisClientRef!.setex).not.toHaveBeenCalled(); }); + it("marks only a confirmed legacy force-update as eligible for legacy cleanup", async () => { + legacyProviderId = 1; + + const result = await SessionManager.updateSessionBindingSmart( + SID, + 2, + 10, + false, + false, + KEY_ID, + true + ); + + expect(result).toMatchObject({ + updated: true, + reason: "race_winner_forced", + legacyBindingUpdated: true, + }); + expect(result.bindingSnapshot).toBeUndefined(); + }); + it.each([ { isFailoverSuccess: false, forceUpdate: true }, { isFailoverSuccess: true, forceUpdate: false }, diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index fabe26ffd..3a8818db1 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -983,7 +983,10 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { const response = await responsePromise; const deferred = peekDeferredStreamingFinalization(session); expect(await response.text()).toContain('"provider":"p2"'); - await expect(deferred?.hedgeBindingSnapshotPromise).resolves.toEqual(winnerBindingSnapshot); + await expect(deferred?.hedgeBindingAuthorityPromise).resolves.toEqual({ + snapshot: winnerBindingSnapshot, + legacyClearAllowed: false, + }); expect(controller1.signal.aborted).toBe(true); expect(controller2.signal.aborted).toBe(false); expect(mocks.recordFailure).not.toHaveBeenCalled(); diff --git a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts index 02cc32cc4..75b5d7f97 100644 --- a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts +++ b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts @@ -601,7 +601,10 @@ describe("Endpoint circuit breaker isolation", () => { } as const; setDeferredMeta(session, 42, { isHedgeWinner: true, - hedgeBindingSnapshotPromise: Promise.resolve(snapshot), + hedgeBindingAuthorityPromise: Promise.resolve({ + snapshot, + legacyClearAllowed: false, + }), }); vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(1_000); const controlled = createControllableSuccessStreamResponse(); @@ -641,7 +644,10 @@ describe("Endpoint circuit breaker isolation", () => { } as const; setDeferredMeta(session, 42, { isHedgeWinner: true, - hedgeBindingSnapshotPromise: Promise.resolve(snapshot), + hedgeBindingAuthorityPromise: Promise.resolve({ + snapshot, + legacyClearAllowed: false, + }), }); vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(1_000); vi.mocked(SessionManager.clearVersionedSessionProvider).mockResolvedValueOnce({ @@ -670,6 +676,52 @@ describe("Endpoint circuit breaker isolation", () => { } }); + it("does not let a failed stale Hedge stream clear a newer same-Provider generation", async () => { + const session = createSession(); + setDeferredMeta(session, 42, { + isHedgeWinner: true, + // The first-byte write read G0, but a concurrent request established + // Provider 1 at G1 before its CAS. No authority over G1 was acquired. + hedgeBindingAuthorityPromise: Promise.resolve({ + snapshot: null, + legacyClearAllowed: false, + }), + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createFake200StreamResponse() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.clearVersionedSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.clearSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.touchVersionedSessionBinding).not.toHaveBeenCalled(); + }); + + it("uses generic cleanup only after the Hedge winner confirms a legacy binding write", async () => { + const session = createSession(); + setDeferredMeta(session, 42, { + isHedgeWinner: true, + hedgeBindingAuthorityPromise: Promise.resolve({ + snapshot: null, + legacyClearAllowed: true, + }), + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createFake200StreamResponse() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.clearVersionedSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1, 456); + expect(SessionManager.touchVersionedSessionBinding).not.toHaveBeenCalled(); + }); + it("stops Hedge heartbeats after a generation conflict and never revives the binding", async () => { vi.useFakeTimers(); try { @@ -682,7 +734,10 @@ describe("Endpoint circuit breaker isolation", () => { } as const; setDeferredMeta(session, 42, { isHedgeWinner: true, - hedgeBindingSnapshotPromise: Promise.resolve(snapshot), + hedgeBindingAuthorityPromise: Promise.resolve({ + snapshot, + legacyClearAllowed: false, + }), }); vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(1_000); vi.mocked(SessionManager.touchVersionedSessionBinding) From 74bb97eaba64684791174c9edf93146099542362 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 21:57:33 -0400 Subject: [PATCH 55/89] fix(discovery): fence readiness and sticky cooldown --- src/app/v1/_lib/proxy/discovery-validity.ts | 39 +++- src/app/v1/_lib/proxy/forwarder.ts | 14 +- tests/unit/proxy/discovery-validity.test.ts | 43 ++++ .../proxy-forwarder-hedge-first-byte.test.ts | 210 ++++++++++++++++++ 4 files changed, 304 insertions(+), 2 deletions(-) diff --git a/src/app/v1/_lib/proxy/discovery-validity.ts b/src/app/v1/_lib/proxy/discovery-validity.ts index 059bde293..1f8931d7d 100644 --- a/src/app/v1/_lib/proxy/discovery-validity.ts +++ b/src/app/v1/_lib/proxy/discovery-validity.ts @@ -49,6 +49,43 @@ function hasAnthropicContentBlock(value: unknown): boolean { return block.type === "text" ? hasContent(block.text) : true; } +function hasOpenAIResponsesOutputItem(value: unknown): boolean { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const item = value as Record; + if (typeof item.type !== "string") return false; + + switch (item.type) { + case "message": + return hasContent(item.content); + case "reasoning": + return hasContent(item.summary) || hasContent(item.content); + case "function_call": + case "mcp_call": + return hasContent(item.name) || hasContent(item.arguments); + case "custom_tool_call": + return hasContent(item.name) || hasContent(item.input); + case "computer_call": + case "web_search_call": + case "file_search_call": + case "code_interpreter_call": + case "local_shell_call": + case "shell_call": + case "apply_patch_call": + return [ + item.action, + item.arguments, + item.input, + item.queries, + item.query, + item.code, + item.command, + item.operation, + ].some(hasContent); + default: + return false; + } +} + function classifyJson(value: unknown, protocol: DiscoveryProtocol): DiscoveryValidity { if (!value || typeof value !== "object") return { ready: false, terminal: false, error: true }; const object = value as Record; @@ -78,7 +115,7 @@ function classifyJson(value: unknown, protocol: DiscoveryProtocol): DiscoveryVal ready: (object.type === "response.output_text.delta" && hasContent(object.delta)) || (object.type === "response.function_call_arguments.delta" && hasContent(object.delta)) || - (object.type === "response.output_item.added" && hasContent(object.item)), + (object.type === "response.output_item.added" && hasOpenAIResponsesOutputItem(object.item)), terminal: false, error: false, }; diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 7f73b8a0b..8016b0502 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -5063,6 +5063,7 @@ export class ProxyForwarder { const roundLaunchIdleWaiters = new Set<() => void>(); let fallbackPromotionBlocked = false; let stickyTimeoutWaveReservation: { fallbackAttemptId: string } | null = null; + let stickyTimeoutCooldownPromise: Promise | null = null; const hasSticky = session.shouldReuseProvider() && !!session.sessionId && @@ -5186,6 +5187,10 @@ export class ProxyForwarder { if (roundTimer) clearTimeout(roundTimer); if (stickyTimer) clearTimeout(stickyTimer); cancelLosers(null, options.cancellationKind ?? "discovery_loser"); + // Sticky timeout owns its cooldown mutation. A terminal deadline/error or + // client abort may race that Redis CAS, but must not issue a second + // zero-cooldown clear against the same captured generation. + if (stickyTimeoutCooldownPromise) await stickyTimeoutCooldownPromise; if (!options.preserveBinding && bindingWriteAllowed && session.isSessionBindingAllowed()) { if (bindingSnapshot.providerId != null) { await SessionManager.clearVersionedSessionProvider( @@ -5338,6 +5343,13 @@ export class ProxyForwarder { } }; + const ensureStickyTimeoutCooldown = (cooldownTtlSeconds: number): Promise => { + if (!stickyTimeoutCooldownPromise) { + stickyTimeoutCooldownPromise = clearCapturedStickyBinding(cooldownTtlSeconds); + } + return stickyTimeoutCooldownPromise; + }; + const scheduleRoundBoundary = (delayMs: number) => { clearRoundTimer(); const epoch = coordinator.epochs; @@ -6019,7 +6031,7 @@ export class ProxyForwarder { fallbackPromotionBlocked = true; stickyTimeoutWaveReservation = { fallbackAttemptId: sticky.id }; if (bindingSnapshot && bindingSnapshot.providerId === initialProvider.id) { - void clearCapturedStickyBinding( + void ensureStickyTimeoutCooldown( Math.ceil((settings.stickyTimeoutCooldownMs ?? 300_000) / 1000) ).finally(() => { void launchReservedStickyTimeoutWave(Math.max(0, concurrency - 1)).catch( diff --git a/tests/unit/proxy/discovery-validity.test.ts b/tests/unit/proxy/discovery-validity.test.ts index 5093298dd..ed2690d6c 100644 --- a/tests/unit/proxy/discovery-validity.test.ts +++ b/tests/unit/proxy/discovery-validity.test.ts @@ -63,6 +63,49 @@ describe("discovery validity", () => { ).toBe(true); }); + it("holds Responses output-item metadata until a text delta is deliverable", () => { + const parser = new DiscoveryValidityParser("openai-responses"); + + expect( + parser.push( + 'data: {"type":"response.output_item.added","item":{"id":"msg_1","type":"message","status":"in_progress","content":[]}}\n\n' + ) + ).toEqual({ ready: false, terminal: false, error: false }); + expect(parser.push('data: {"type":"response.output_text.delta","delta":"hello"}\n\n')).toEqual({ + ready: true, + terminal: false, + error: false, + }); + }); + + it("does not let Responses output-item metadata mask a later error", () => { + const parser = new DiscoveryValidityParser("openai-responses"); + + expect( + parser.push( + 'data: {"type":"response.output_item.added","item":{"id":"msg_1","type":"message","status":"in_progress"}}\n\n' + ) + ).toMatchObject({ ready: false, error: false }); + expect( + parser.push('data: {"type":"response.failed","error":{"message":"upstream failed"}}\n\n') + ).toEqual({ ready: false, terminal: true, error: true }); + }); + + it("accepts only an explicit non-empty Responses tool payload", () => { + expect( + classifyDiscoveryChunk( + 'data: {"type":"response.output_item.added","item":{"id":"fc_1","type":"function_call","status":"in_progress"}}\n', + "openai-responses" + ).ready + ).toBe(false); + expect( + classifyDiscoveryChunk( + 'data: {"type":"response.output_item.added","item":{"id":"fc_1","type":"function_call","name":"lookup","arguments":"{}"}}\n', + "openai-responses" + ).ready + ).toBe(true); + }); + it("consumes split SSE lines incrementally without waiting for the full stream", () => { const parser = new DiscoveryValidityParser("openai-chat"); expect(parser.push('data: {"choices":[{"delta":{"content":"hel')).toEqual({ diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index 225e382db..9f37194bc 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -2640,6 +2640,216 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { } }); + test("Sticky timeout cooldown completes once before a racing total deadline settles", async () => { + vi.useFakeTimers(); + try { + const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 32 }, + apiKey: null, + } as typeof session.authState; + session.request.message.messages = [ + { role: "user", content: "first" }, + { role: "user", content: "second" }, + ]; + session.setProvider(sticky); + session.setSessionBindingSnapshot({ + sessionId: session.sessionId!, + keyId: 32, + providerId: sticky.id, + generation: "g-sticky-cooldown-deadline", + }); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 20, + stickySlaMs: 10, + racingTotalTimeoutMs: 30, + stickyTimeoutCooldownMs: 300_000, + }); + + const cooldownClear = Promise.withResolvers<{ + status: "ok"; + legacyFallbackAllowed: false; + source: "cleared"; + snapshot: { + sessionId: string; + keyId: number; + providerId: null; + generation: string; + }; + }>(); + const order: string[] = []; + mocks.clearVersionedSessionProvider.mockImplementationOnce( + async (_snapshot, _providerId, cooldownTtlSeconds) => { + order.push(`cooldown-start:${cooldownTtlSeconds}`); + const result = await cooldownClear.promise; + order.push("cooldown-end"); + return result; + } + ); + mocks.releaseSessionDiscoveryLease.mockImplementationOnce(async () => { + order.push("lease-release"); + return { status: "released", legacyFallbackAllowed: false }; + }); + vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ).mockResolvedValueOnce( + new Response(new ReadableStream(), { + headers: { "content-type": "text/event-stream" }, + }) + ); + + let requestSettled = false; + const observed = ProxyForwarder.send(session).then( + (response) => { + requestSettled = true; + return response; + }, + (error) => { + requestSettled = true; + return error; + } + ); + + await vi.advanceTimersByTimeAsync(10); + expect(mocks.clearVersionedSessionProvider).toHaveBeenCalledWith( + expect.objectContaining({ generation: "g-sticky-cooldown-deadline" }), + sticky.id, + 300 + ); + expect(order).toEqual(["cooldown-start:300"]); + + await vi.advanceTimersByTimeAsync(20); + expect(requestSettled).toBe(false); + expect(mocks.clearVersionedSessionProvider).toHaveBeenCalledOnce(); + expect(mocks.releaseSessionDiscoveryLease).not.toHaveBeenCalled(); + + cooldownClear.resolve({ + status: "ok", + legacyFallbackAllowed: false, + source: "cleared", + snapshot: { + sessionId: session.sessionId!, + keyId: 32, + providerId: null, + generation: "g-sticky-cooldown-applied", + }, + }); + await vi.advanceTimersByTimeAsync(0); + + expect(await observed).toBeInstanceOf(UpstreamProxyError); + expect(mocks.clearVersionedSessionProvider).toHaveBeenCalledOnce(); + expect(mocks.pickDiscoveryProviders).not.toHaveBeenCalled(); + expect(order).toEqual(["cooldown-start:300", "cooldown-end", "lease-release"]); + } finally { + vi.useRealTimers(); + } + }); + + test("client abort preserves an already-reserved Sticky timeout cooldown", async () => { + vi.useFakeTimers(); + try { + const clientAbort = new AbortController(); + const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); + const session = createSession(clientAbort.signal); + session.authState = { + success: true, + user: null, + key: { id: 33 }, + apiKey: null, + } as typeof session.authState; + session.request.message.messages = [ + { role: "user", content: "first" }, + { role: "user", content: "second" }, + ]; + session.setProvider(sticky); + session.setSessionBindingSnapshot({ + sessionId: session.sessionId!, + keyId: 33, + providerId: sticky.id, + generation: "g-sticky-cooldown-abort", + }); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 1, + discoverySlaMs: 50, + stickySlaMs: 10, + racingTotalTimeoutMs: 100, + stickyTimeoutCooldownMs: 300_000, + }); + + const cooldownClear = Promise.withResolvers<{ + status: "ok"; + legacyFallbackAllowed: false; + source: "cleared"; + snapshot: { + sessionId: string; + keyId: number; + providerId: null; + generation: string; + }; + }>(); + mocks.clearVersionedSessionProvider.mockReturnValueOnce(cooldownClear.promise); + vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ).mockResolvedValueOnce( + new Response(new ReadableStream(), { + headers: { "content-type": "text/event-stream" }, + }) + ); + + let requestSettled = false; + const observed = ProxyForwarder.send(session).catch((error) => { + requestSettled = true; + return error; + }); + await vi.advanceTimersByTimeAsync(10); + clientAbort.abort(new Error("client disconnected")); + await vi.advanceTimersByTimeAsync(0); + + expect(requestSettled).toBe(false); + expect(mocks.clearVersionedSessionProvider).toHaveBeenCalledOnce(); + expect(mocks.clearVersionedSessionProvider).toHaveBeenCalledWith( + expect.objectContaining({ generation: "g-sticky-cooldown-abort" }), + sticky.id, + 300 + ); + + cooldownClear.resolve({ + status: "ok", + legacyFallbackAllowed: false, + source: "cleared", + snapshot: { + sessionId: session.sessionId!, + keyId: 33, + providerId: null, + generation: "g-sticky-cooldown-abort-applied", + }, + }); + await vi.advanceTimersByTimeAsync(0); + + const error = await observed; + expect(error).toBeInstanceOf(UpstreamProxyError); + expect((error as UpstreamProxyError).statusCode).toBe(499); + expect(mocks.clearVersionedSessionProvider).toHaveBeenCalledOnce(); + expect(mocks.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); + } finally { + vi.useRealTimers(); + } + }); + test("a ready-held Sticky fallback survives a stalled next-wave selector until the total deadline", async () => { vi.useFakeTimers(); try { From 684e334e5b229b283d5f965e07a311ef982eca36 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 22:06:33 -0400 Subject: [PATCH 56/89] fix(session): scope content hash mapping by API key --- src/lib/session-manager.ts | 72 ++++++- .../lib/session-manager-content-hash.test.ts | 198 ++++++++++++++++++ 2 files changed, 263 insertions(+), 7 deletions(-) create mode 100644 tests/unit/lib/session-manager-content-hash.test.ts diff --git a/src/lib/session-manager.ts b/src/lib/session-manager.ts index cd829b620..68df0b412 100644 --- a/src/lib/session-manager.ts +++ b/src/lib/session-manager.ts @@ -222,6 +222,10 @@ function parseSessionDetailResponseMeta(value: string): SessionDetailResponseMet } } +function buildTenantContentHashSessionKey(keyId: number, contentHash: string): string { + return `hash:${keyId}:${contentHash}:session`; +} + /** * Session 管理器 * @@ -313,6 +317,42 @@ export class SessionManager { return `sess_${timestamp}_${random}`; } + private static async proveContentHashSessionOwnership( + redis: NonNullable>, + sessionId: string, + keyId: number + ): Promise<{ owned: boolean; ownerPresent: boolean }> { + const legacyOwner = await redis.get(`session:${sessionId}:key`); + if (legacyOwner !== keyId.toString()) { + return { owned: false, ownerPresent: legacyOwner !== null }; + } + + // Reconcile only after the legacy owner proves the tenant. This both + // validates canonical/mirror consistency and refreshes the complete binding + // TTL, preventing an owner-expiry race between hash lookup and Provider selection. + const binding = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status === "ok") { + return { owned: true, ownerPresent: true }; + } + if (!binding.legacyFallbackAllowed) { + return { owned: false, ownerPresent: true }; + } + + const legacyRefresh = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "refresh" }, + }); + return { owned: legacyRefresh.status === "ok", ownerPresent: true }; + } + /** * 获取 Session 内下一个请求序号（原子操作） * @@ -545,17 +585,32 @@ export class SessionManager { // 3. 尝试从 Redis 查找已有 session if (redis && redis.status === "ready") { try { - const hashKey = `hash:${contentHash}:session`; + const hashKey = buildTenantContentHashSessionKey(keyId, contentHash); const existingSessionId = await redis.get(hashKey); if (existingSessionId) { - // 找到已有 session，刷新 TTL - await SessionManager.refreshSessionTTL(existingSessionId, keyId); - logger.trace("SessionManager: Reusing session via hash", { - sessionId: existingSessionId, + const ownership = await SessionManager.proveContentHashSessionOwnership( + redis, + existingSessionId, + keyId + ); + if (ownership.owned) { + // 找到当前 tenant 的已有 session，刷新 TTL + await SessionManager.refreshSessionTTL(existingSessionId, keyId); + logger.trace("SessionManager: Reusing tenant-scoped session via hash", { + sessionId: existingSessionId, + hash: contentHash, + keyId, + }); + return existingSessionId; + } + + logger.warn("SessionManager: Ignoring content-hash mapping without matching owner", { hash: contentHash, + keyId, + mappingScope: "tenant", + ownerPresent: ownership.ownerPresent, }); - return existingSessionId; } // 未找到：创建新 session @@ -611,7 +666,10 @@ export class SessionManager { } const pipeline = redis.pipeline(); - const hashKey = `hash:${contentHash}:session`; + // Do not dual-write the historical unscoped key. Mixed-version workers + // may temporarily create separate Sessions; old mappings expire naturally + // without allowing the new path to import tenant-ambiguous state. + const hashKey = buildTenantContentHashSessionKey(keyId, contentHash); // 存储映射关系 pipeline.setex(hashKey, SessionManager.SESSION_TTL, sessionId); diff --git a/tests/unit/lib/session-manager-content-hash.test.ts b/tests/unit/lib/session-manager-content-hash.test.ts new file mode 100644 index 000000000..1cd2c393c --- /dev/null +++ b/tests/unit/lib/session-manager-content-hash.test.ts @@ -0,0 +1,198 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +let redisClientRef: { + status: string; + get: ReturnType; + setex: ReturnType; + pipeline: ReturnType; +}; +let values: Map; + +const bindingMocks = vi.hoisted(() => ({ + mutateLegacySessionBindingSafely: vi.fn(), + readOrReconcileSessionBinding: vi.fn(), +})); + +vi.mock("server-only", () => ({})); + +vi.mock("@/lib/logger", () => ({ + logger: { + debug: vi.fn(), + error: vi.fn(), + info: vi.fn(), + trace: vi.fn(), + warn: vi.fn(), + }, +})); + +vi.mock("@/lib/redis", () => ({ + getRedisClient: () => redisClientRef, +})); + +vi.mock("@/lib/redis/session-binding", () => ({ + ...bindingMocks, + getVersionedBindingCapabilityState: () => "available", +})); + +vi.mock("@/lib/session-tracker", () => ({ + SessionTracker: { + getConcurrentCount: vi.fn(async () => 0), + }, +})); + +import { SessionManager } from "@/lib/session-manager"; + +const MESSAGES = [{ role: "user", content: "identical tenant-sensitive prompt" }]; + +function contentHash(): string { + const hash = SessionManager.calculateMessagesHash(MESSAGES); + if (!hash) throw new Error("Expected test messages to produce a content hash"); + return hash; +} + +function tenantHashKey(keyId: number): string { + return `hash:${keyId}:${contentHash()}:session`; +} + +function legacyHashKey(): string { + return `hash:${contentHash()}:session`; +} + +beforeEach(() => { + vi.clearAllMocks(); + values = new Map(); + + const createPipeline = () => { + const operations: Array<() => void> = []; + const pipeline = { + setex: vi.fn((key: string, _ttlSeconds: number, value: string) => { + operations.push(() => values.set(key, value)); + return pipeline; + }), + exec: vi.fn(async () => { + for (const operation of operations) operation(); + return operations.map(() => [null, "OK"]); + }), + }; + return pipeline; + }; + + redisClientRef = { + status: "ready", + get: vi.fn(async (key: string) => values.get(key) ?? null), + setex: vi.fn(async (key: string, _ttlSeconds: number, value: string) => { + values.set(key, value); + return "OK"; + }), + pipeline: vi.fn(createPipeline), + }; + + bindingMocks.readOrReconcileSessionBinding.mockImplementation( + async ({ sessionId, keyId }: { sessionId: string; keyId: number }) => { + values.set(`session:${sessionId}:key`, keyId.toString()); + return { + status: "ok", + source: "created", + snapshot: { + sessionId, + keyId, + providerId: null, + generation: `generation-${keyId}`, + }, + legacyFallbackAllowed: false, + }; + } + ); +}); + +describe("SessionManager content-hash mapping tenant isolation", () => { + it("does not share a generated Session between API keys with identical content", async () => { + const first = await SessionManager.getOrCreateSessionId(101, MESSAGES, null); + await vi.waitFor(() => expect(values.get(tenantHashKey(101))).toBe(first)); + + const second = await SessionManager.getOrCreateSessionId(202, MESSAGES, null); + await vi.waitFor(() => expect(values.get(tenantHashKey(202))).toBe(second)); + + expect(second).not.toBe(first); + expect(values.has(legacyHashKey())).toBe(false); + }); + + it("continues to reuse the tenant-scoped mapping for the same API key", async () => { + const first = await SessionManager.getOrCreateSessionId(101, MESSAGES, null); + await vi.waitFor(() => expect(values.get(tenantHashKey(101))).toBe(first)); + + bindingMocks.readOrReconcileSessionBinding.mockClear(); + const second = await SessionManager.getOrCreateSessionId(101, MESSAGES, null); + + expect(second).toBe(first); + expect(bindingMocks.readOrReconcileSessionBinding).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: first, keyId: 101 }) + ); + }); + + it.each([ + ["matching", "101"], + ["foreign", "202"], + ["missing", null], + ])("never reads or imports an unscoped legacy mapping with a %s owner", async (_case, owner) => { + const legacySessionId = `legacy-${_case}-owner-session`; + values.set(legacyHashKey(), legacySessionId); + if (owner !== null) values.set(`session:${legacySessionId}:key`, owner); + + const result = await SessionManager.getOrCreateSessionId(101, MESSAGES, null); + + expect(result).not.toBe(legacySessionId); + await vi.waitFor(() => expect(values.get(tenantHashKey(101))).toBe(result)); + expect(redisClientRef.get).not.toHaveBeenCalledWith(legacyHashKey()); + expect(values.get(legacyHashKey())).toBe(legacySessionId); + expect(values.get(`session:${legacySessionId}:key`)).toBe(owner ?? undefined); + }); + + it.each([ + ["missing", null], + ["foreign", "202"], + ["corrupt", "not-a-key-id"], + ])("rejects a tenant-scoped mapping with a %s owner", async (_case, owner) => { + const staleSessionId = `scoped-${_case}-owner-session`; + values.set(tenantHashKey(101), staleSessionId); + if (owner !== null) values.set(`session:${staleSessionId}:key`, owner); + + const result = await SessionManager.getOrCreateSessionId(101, MESSAGES, null); + + expect(result).not.toBe(staleSessionId); + await vi.waitFor(() => expect(values.get(tenantHashKey(101))).toBe(result)); + expect(values.get(`session:${staleSessionId}:key`)).toBe(owner ?? undefined); + }); + + it("rejects a tenant-scoped mapping when canonical reconciliation fails", async () => { + const staleSessionId = "scoped-conflicting-binding-session"; + values.set(tenantHashKey(101), staleSessionId); + values.set(`session:${staleSessionId}:key`, "101"); + bindingMocks.readOrReconcileSessionBinding.mockResolvedValueOnce({ + status: "conflict", + reason: "mirror_mismatch", + legacyFallbackAllowed: false, + }); + + const result = await SessionManager.getOrCreateSessionId(101, MESSAGES, null); + + expect(result).not.toBe(staleSessionId); + await vi.waitFor(() => expect(values.get(tenantHashKey(101))).toBe(result)); + }); + + it("does not publish a tenant-scoped mapping when binding initialization fails", async () => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValueOnce({ + status: "conflict", + reason: "foreign_owner", + legacyFallbackAllowed: false, + }); + + await SessionManager.getOrCreateSessionId(101, MESSAGES, null); + + await vi.waitFor(() => { + expect(bindingMocks.readOrReconcileSessionBinding).toHaveBeenCalledTimes(1); + }); + expect(values.has(tenantHashKey(101))).toBe(false); + expect(redisClientRef.pipeline).not.toHaveBeenCalled(); + }); +}); From 088061992968c39ee78116af91c52571b8a8a7f8 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 22:17:52 -0400 Subject: [PATCH 57/89] fix(discovery): unwrap Gemini response candidates --- src/app/v1/_lib/proxy/discovery-validity.ts | 7 ++++++- tests/unit/proxy/discovery-validity.test.ts | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/app/v1/_lib/proxy/discovery-validity.ts b/src/app/v1/_lib/proxy/discovery-validity.ts index 7686c6bf6..3e67cc523 100644 --- a/src/app/v1/_lib/proxy/discovery-validity.ts +++ b/src/app/v1/_lib/proxy/discovery-validity.ts @@ -143,7 +143,12 @@ function classifyJson(value: unknown, protocol: DiscoveryProtocol): DiscoveryVal }; } if (protocol === "gemini") { - const candidates = Array.isArray(object.candidates) ? object.candidates : []; + const response = + object.response && typeof object.response === "object" && !Array.isArray(object.response) + ? (object.response as Record) + : null; + const candidatesValue = response?.candidates ?? object.candidates; + const candidates = Array.isArray(candidatesValue) ? candidatesValue : []; return { ready: candidates.some((candidate) => hasContent(candidate)), terminal: false, diff --git a/tests/unit/proxy/discovery-validity.test.ts b/tests/unit/proxy/discovery-validity.test.ts index 76c8cc192..00dccd2f8 100644 --- a/tests/unit/proxy/discovery-validity.test.ts +++ b/tests/unit/proxy/discovery-validity.test.ts @@ -27,6 +27,21 @@ describe("discovery validity", () => { expect(classifyDiscoveryChunk("data: [DONE]\n", "openai-chat").terminal).toBe(true); }); + it("accepts Gemini candidates in the supported response wrapper", () => { + expect( + classifyDiscoveryChunk( + '{"response":{"candidates":[{"content":{"parts":[{"text":"hello"}]}}]}}', + "gemini" + ) + ).toEqual({ ready: true, terminal: false, error: false }); + }); + + it("keeps wrapped Gemini errors terminal", () => { + expect( + classifyDiscoveryChunk('{"response":{"error":{"message":"upstream failed"}}}', "gemini") + ).toEqual({ ready: false, terminal: true, error: true }); + }); + it("rejects errors even when a later chunk contains content", () => { const parser = new DiscoveryValidityParser("openai-responses"); expect(parser.push('{"type":"response.failed","error":{"message":"no"}}').error).toBe(true); From 8b1b678bd9a548791859c09467803c73eef16a6a Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 22:26:40 -0400 Subject: [PATCH 58/89] fix(binding): avoid reusing canonical legacy mirrors --- src/lib/session-manager.ts | 13 ++++++ .../session-manager-versioned-binding.test.ts | 46 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/src/lib/session-manager.ts b/src/lib/session-manager.ts index 68df0b412..f96169f0e 100644 --- a/src/lib/session-manager.ts +++ b/src/lib/session-manager.ts @@ -33,6 +33,7 @@ import { } from "./redis/active-session-keys"; import { acquireSessionDiscoveryLease as acquireVersionedSessionDiscoveryLease, + buildSessionBindingKeys, clearSessionBinding as clearVersionedSessionBinding, compareAndSetSessionBinding, ensureVersionedBindingCapability, @@ -980,6 +981,18 @@ export class SessionManager { return null; } + // A capability failure may permit a legacy read only when this + // session has no canonical binding. If canonical state exists, the + // legacy mirror is not safely writable and must not be reused. + const bindingKeys = buildSessionBindingKeys(sessionId, keyId); + if ((await redis.exists(bindingKeys.canonical)) > 0) { + logger.warn("SessionManager: Refusing legacy provider reuse with canonical binding", { + sessionId, + keyId, + }); + return null; + } + const boundKeyId = await redis.get(`session:${sessionId}:key`); // Fail-closed：boundKeyId 缺失（TTL 漂移、旧绑定或写入路径未原子写 key）也视为校验失败， // 避免无法证明归属当前 key 的旧 provider binding 继续被复用。 diff --git a/tests/unit/lib/session-manager-versioned-binding.test.ts b/tests/unit/lib/session-manager-versioned-binding.test.ts index af39e3c7e..ba09c068e 100644 --- a/tests/unit/lib/session-manager-versioned-binding.test.ts +++ b/tests/unit/lib/session-manager-versioned-binding.test.ts @@ -2,6 +2,11 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const bindingMocks = vi.hoisted(() => ({ acquireSessionDiscoveryLease: vi.fn(), + buildSessionBindingKeys: vi.fn((sessionId: string, _keyId: number) => ({ + canonical: `session-binding:v1:{${"a".repeat(64)}}:binding`, + legacyProvider: `session:${sessionId}:provider`, + legacyOwner: `session:${sessionId}:key`, + })), clearSessionBinding: vi.fn(), compareAndSetSessionBinding: vi.fn(), isSessionProviderCoolingDown: vi.fn(), @@ -16,6 +21,7 @@ let redisClientRef: { status: string; del: ReturnType; eval: ReturnType; + exists: ReturnType; get: ReturnType; pipeline: ReturnType; set: ReturnType; @@ -73,6 +79,7 @@ beforeEach(() => { status: "ready", del: vi.fn(async () => 1), eval: vi.fn(async () => 1), + exists: vi.fn(async () => 0), get: vi.fn(async () => null), pipeline: vi.fn(() => pipeline), set: vi.fn(async () => "OK"), @@ -181,6 +188,45 @@ describe("SessionManager versioned binding adapter", () => { expect(redisClientRef.get).not.toHaveBeenCalled(); }); + it("does not reuse a legacy mirror when canonical state exists during capability fallback", async () => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "unavailable", + reason: "capability_unavailable", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + }); + redisClientRef.exists.mockResolvedValue(1); + redisClientRef.get.mockResolvedValue(String(PROVIDER_ID)); + + await expect(SessionManager.getSessionProvider(SESSION_ID, KEY_ID)).resolves.toBeNull(); + + expect(redisClientRef.exists).toHaveBeenCalledWith( + expect.stringMatching(/^session-binding:v1:\{[a-f0-9]+\}:binding$/) + ); + expect(redisClientRef.get).not.toHaveBeenCalled(); + }); + + it("still reuses a tenant-owned legacy-only mirror during capability fallback", async () => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "unavailable", + reason: "capability_unavailable", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + }); + redisClientRef.exists.mockResolvedValue(0); + redisClientRef.get.mockImplementation(async (key: string) => + key.endsWith(":key") ? String(KEY_ID) : String(PROVIDER_ID) + ); + + await expect(SessionManager.getSessionProvider(SESSION_ID, KEY_ID)).resolves.toBe(PROVIDER_ID); + + expect(redisClientRef.exists).toHaveBeenCalledWith( + expect.stringMatching(/^session-binding:v1:\{[a-f0-9]+\}:binding$/) + ); + expect(redisClientRef.get).toHaveBeenCalledWith(`session:${SESSION_ID}:key`); + expect(redisClientRef.get).toHaveBeenCalledWith(`session:${SESSION_ID}:provider`); + }); + it("clears through generation CAS and does not delete the legacy provider directly", async () => { bindingMocks.clearSessionBinding.mockResolvedValue({ status: "ok", From 1d23d6cbc195cbfb9230dc7a079ccb555c462443 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 22:33:31 -0400 Subject: [PATCH 59/89] fix(discovery): release non-SSE winner resources --- src/app/v1/_lib/proxy/response-handler.ts | 23 +++++++ ...esponse-handler-client-abort-drain.test.ts | 66 ++++++++++++++++++- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index f7608e16a..94f41e043 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -283,6 +283,25 @@ async function releaseOwnedProviderSessionRef( } } +async function finalizeNonStreamDiscoveryResources( + session: ProxySession, + lifecycle: DiscoveryLeaseLifecycle +): Promise { + const deferred = peekDeferredStreamingFinalization(session); + if (!deferred?.discoveryLease) return; + + // Non-SSE responses do not run the protocol completion finalizer, so they + // cannot safely create or renew Sticky. Consume the metadata and release + // both request-scoped resources after body processing instead of waiting + // for the Redis lease TTL. + const meta = consumeDeferredStreamingFinalization(session); + try { + await releaseOwnedProviderSessionRef(session, meta, false); + } finally { + await lifecycle.release(); + } +} + function startHedgeBindingHeartbeat(session: ProxySession): void { const deferred = peekDeferredStreamingFinalization(session); const authorityPromise = deferred?.hedgeBindingAuthorityPromise; @@ -2185,9 +2204,11 @@ export class ProxyResponseHandler { ): Promise { const messageContext = session.messageContext; const provider = session.provider; + const discoveryLeaseLifecycle = startDiscoveryLeaseLifecycle(session); if (!provider) { discardBeforeResponseBodySnapshot(session); releaseSessionAgent(session); + void finalizeNonStreamDiscoveryResources(session, discoveryLeaseLifecycle); return response; } @@ -2458,6 +2479,7 @@ export class ProxyResponseHandler { } } finally { cleanupTaskAbortBinding(); + await finalizeNonStreamDiscoveryResources(session, discoveryLeaseLifecycle); releaseSessionAgent(session); } }; @@ -3076,6 +3098,7 @@ export class ProxyResponseHandler { } finally { cleanupTaskAbortBinding(); cleanupClientAbortListener(); + await finalizeNonStreamDiscoveryResources(session, discoveryLeaseLifecycle); releaseSessionAgent(session); } }; diff --git a/tests/unit/proxy/response-handler-client-abort-drain.test.ts b/tests/unit/proxy/response-handler-client-abort-drain.test.ts index d52cb370e..17a125f67 100644 --- a/tests/unit/proxy/response-handler-client-abort-drain.test.ts +++ b/tests/unit/proxy/response-handler-client-abort-drain.test.ts @@ -5,7 +5,10 @@ import { ProxyResponseHandler, } from "@/app/v1/_lib/proxy/response-handler"; import { ProxySession } from "@/app/v1/_lib/proxy/session"; -import { setDeferredStreamingFinalization } from "@/app/v1/_lib/proxy/stream-finalization"; +import { + peekDeferredStreamingFinalization, + setDeferredStreamingFinalization, +} from "@/app/v1/_lib/proxy/stream-finalization"; import { AsyncTaskManager, shutdownAllAsyncTasks } from "@/lib/async-task-manager"; import { recordFailure } from "@/lib/circuit-breaker"; import { emitProxyLangfuseTrace } from "@/lib/langfuse/emit-proxy-trace"; @@ -3187,6 +3190,67 @@ describe("ProxyResponseHandler stream client abort finalization", () => { ); }); + it("releases Discovery resources for a non-SSE Gemini winner", async () => { + const session = createSession(new AbortController().signal, { + providerType: "gemini", + originalFormat: "gemini", + endpoint: "/v1beta/models/gemini-2.0-flash:streamGenerateContent", + model: "gemini-2.0-flash", + }); + session.sessionId = "non-sse-gemini-discovery"; + session.recordProviderSessionRef(1); + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "gemini-discovery", + providerPriority: 1, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.invalid/v1", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: { + sessionId: "non-sse-gemini-discovery", + keyId: 2, + providerId: null, + generation: "non-sse-generation", + }, + requiresCompletionMarker: true, + discoveryLease: { + sessionId: "non-sse-gemini-discovery", + keyId: 2, + ownerToken: "non-sse-owner", + ttlSeconds: 30, + }, + providerSessionRefOwned: true, + providerSessionRefRetainOnSuccess: true, + }); + const response = new Response( + '{"response":{"candidates":[{"content":{"parts":[{"text":"hello"}]}}]}}', + { status: 200, headers: { "content-type": "application/json" } } + ); + + const returned = await ProxyResponseHandler.dispatch(session, response); + expect(returned).toBe(response); + await drainAsyncTasks(); + + expect(SessionManager.renewSessionDiscoveryLease).toHaveBeenCalled(); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); + expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledWith( + "non-sse-gemini-discovery", + 2, + "non-sse-owner" + ); + expect(RateLimitService.releaseProviderSession).toHaveBeenCalledWith( + 1, + "non-sse-gemini-discovery" + ); + expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); + expect(peekDeferredStreamingFinalization(session)).toBeNull(); + }); + it("persists one durable 502 before Provider circuit mutation on non-stream response timeout", async () => { const durableAck = createDeferred(); vi.mocked(updateMessageRequestDetailsDurably).mockImplementationOnce( From bde3d9f206854a25e32816104295062841b255a3 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Mon, 20 Jul 2026 22:54:51 -0400 Subject: [PATCH 60/89] fix(discovery): localize validation and completion checks --- messages/en/settings/config.json | 1 + messages/ja/settings/config.json | 1 + messages/ru/settings/config.json | 1 + messages/zh-CN/settings/config.json | 1 + messages/zh-TW/settings/config.json | 1 + src/actions/system-config.ts | 25 ++++++-- .../_components/system-settings-form.tsx | 57 ++++++++++++++----- src/app/api/v1/resources/system/handlers.ts | 5 +- src/app/api/v1/resources/system/router.ts | 9 ++- src/app/v1/_lib/proxy/response-handler.ts | 6 +- src/lib/api/v1/_shared/error-envelope.ts | 4 +- src/lib/api/v1/_shared/request-body.ts | 9 ++- src/lib/api/v1/schemas/system-config.ts | 38 +++++++++---- src/lib/validation/discovery-settings.ts | 28 +++++++++ src/lib/validation/schemas.ts | 48 +++++++++------- tests/api/v1/system/system-config.test.ts | 15 +++++ tests/unit/actions/system-config-save.test.ts | 27 +++++++++ ...handler-endpoint-circuit-isolation.test.ts | 7 +++ .../system-settings-discovery.test.ts | 13 ++++- 19 files changed, 239 insertions(+), 57 deletions(-) create mode 100644 src/lib/validation/discovery-settings.ts diff --git a/messages/en/settings/config.json b/messages/en/settings/config.json index ad2fe21e8..149408b5e 100644 --- a/messages/en/settings/config.json +++ b/messages/en/settings/config.json @@ -128,6 +128,7 @@ "stickyTimeoutCooldownMs": "Sticky timeout cooldown (milliseconds)", "discoveryWindowDesc": "The total timeout must be at least Sticky SLA + maximum rounds × Discovery SLA. Discovery losers are cancelled and are not drained or billed by the legacy Hedge path.", "discoveryWindowInvalid": "Discovery total timeout is shorter than the configured Sticky and Discovery windows.", + "discoverySettingsInvalid": "One or more Discovery values are outside the allowed range.", "verboseProviderError": "Verbose Provider Error", "verboseProviderErrorDesc": "When enabled, CCH may return detailed diagnostic information for some upstream failure types in `error.details` (for example provider availability diagnostics or sanitized upstream snippets).", "verboseProviderErrorTooltip": "May expose provider names, internal routing clues, upstream failure reasons, and other diagnostic details. Enable only if clients are allowed to see low-level troubleshooting context.", diff --git a/messages/ja/settings/config.json b/messages/ja/settings/config.json index b7ca0000b..e49d28c1b 100644 --- a/messages/ja/settings/config.json +++ b/messages/ja/settings/config.json @@ -130,6 +130,7 @@ "stickyTimeoutCooldownMs": "Sticky タイムアウト後のクールダウン（ミリ秒）", "discoveryWindowDesc": "合計タイムアウトは Sticky SLA + 最大ラウンド数 × Discovery SLA 以上にしてください。", "discoveryWindowInvalid": "Discovery 合計タイムアウトが設定された Sticky/Discovery ウィンドウより短くなっています。", + "discoverySettingsInvalid": "1 つ以上の Discovery 設定値が許容範囲外です。", "verboseProviderError": "詳細なプロバイダーエラー", "verboseProviderErrorDesc": "有効にすると、一部の上流障害タイプで `error.details` により詳細な診断情報（プロバイダー可用性の診断やサニタイズ済み上流断片など）を含める場合があります。", "verboseProviderErrorTooltip": "この設定を有効にすると、プロバイダー名、内部ルーティングの手掛かり、上流障害の理由などの診断情報が露出する可能性があります。クライアントに低レベルのトラブルシュート文脈を見せてもよい場合にのみ有効化してください。", diff --git a/messages/ru/settings/config.json b/messages/ru/settings/config.json index 108a2e4f8..9624cf448 100644 --- a/messages/ru/settings/config.json +++ b/messages/ru/settings/config.json @@ -130,6 +130,7 @@ "stickyTimeoutCooldownMs": "Пауза после тайм-аута Sticky (миллисекунды)", "discoveryWindowDesc": "Общий тайм-аут должен быть не меньше SLA Sticky + максимальное число раундов × SLA Discovery.", "discoveryWindowInvalid": "Общий тайм-аут Discovery меньше настроенного окна Sticky и Discovery.", + "discoverySettingsInvalid": "Одно или несколько значений Discovery находятся вне допустимого диапазона.", "verboseProviderError": "Подробные ошибки провайдеров", "verboseProviderErrorDesc": "При включении CCH может добавлять более подробную диагностику некоторых типов сбоев апстрима в `error.details` (например, диагностику доступности провайдеров или очищенные фрагменты ответа апстрима).", "verboseProviderErrorTooltip": "Может раскрывать названия провайдеров, внутренние подсказки маршрутизации, причины сбоев апстрима и другие диагностические детали. Включайте только если клиентам допустимо видеть низкоуровневый контекст отладки.", diff --git a/messages/zh-CN/settings/config.json b/messages/zh-CN/settings/config.json index b3249e8dd..c4a94ea93 100644 --- a/messages/zh-CN/settings/config.json +++ b/messages/zh-CN/settings/config.json @@ -57,6 +57,7 @@ "stickyTimeoutCooldownMs": "Sticky 超时冷却（毫秒）", "discoveryWindowDesc": "总超时必须不小于 Sticky SLA + 最大轮数 × Discovery SLA。Discovery 输家会取消，不走旧 Hedge 的 drain 或输家计费。", "discoveryWindowInvalid": "Discovery 总超时短于已配置的 Sticky 与 Discovery 窗口。", + "discoverySettingsInvalid": "一个或多个 Discovery 配置值超出允许范围。", "verboseProviderError": "详细供应商错误信息", "verboseProviderErrorDesc": "开启后，CCH 会在某些上游失败类型下于 `error.details` 返回更详细的诊断信息（例如供应商可用性诊断或脱敏后的上游片段）。", "verboseProviderErrorTooltip": "该选项可能暴露供应商名称、内部路由线索、上游失败原因等诊断信息。仅建议在客户端可以查看底层排障上下文时开启。", diff --git a/messages/zh-TW/settings/config.json b/messages/zh-TW/settings/config.json index 8ccc6318a..613494c92 100644 --- a/messages/zh-TW/settings/config.json +++ b/messages/zh-TW/settings/config.json @@ -130,6 +130,7 @@ "stickyTimeoutCooldownMs": "Sticky 逾時冷卻（毫秒）", "discoveryWindowDesc": "總逾時必須不小於 Sticky SLA + 最大輪數 × Discovery SLA。Discovery 輸家會取消，不走舊 Hedge 的 drain 或輸家計費。", "discoveryWindowInvalid": "Discovery 總逾時短於已設定的 Sticky 與 Discovery 視窗。", + "discoverySettingsInvalid": "一個或多個 Discovery 設定值超出允許範圍。", "verboseProviderError": "詳細供應商錯誤資訊", "verboseProviderErrorDesc": "開啟後，CCH 會在某些上游失敗類型下於 `error.details` 返回較詳細的診斷資訊（例如供應商可用性診斷或脫敏後的上游片段）。", "verboseProviderErrorTooltip": "此選項可能暴露供應商名稱、內部路由線索、上游失敗原因等診斷資訊。僅建議在客戶端可以查看底層排障上下文時開啟。", diff --git a/src/actions/system-config.ts b/src/actions/system-config.ts index 40e46320b..4ffbdf87c 100644 --- a/src/actions/system-config.ts +++ b/src/actions/system-config.ts @@ -1,6 +1,7 @@ "use server"; import { revalidatePath } from "next/cache"; +import { ZodError } from "zod"; import { locales } from "@/i18n/config"; import { emitActionAudit } from "@/lib/audit/emit"; import { getSession } from "@/lib/auth"; @@ -15,6 +16,10 @@ import { invalidateAllStatisticsCaches, } from "@/lib/redis"; import { resolveSystemTimezone } from "@/lib/utils/timezone"; +import { + DISCOVERY_WINDOW_INVALID_ERROR_CODE, + getDiscoveryValidationErrorCode, +} from "@/lib/validation/discovery-settings"; import { UpdateSystemSettingsSchema } from "@/lib/validation/schemas"; import { getSystemSettings, updateSystemSettings } from "@/repository/system-config"; import type { IpExtractionConfig } from "@/types/ip-extraction"; @@ -26,7 +31,10 @@ import type { } from "@/types/system-config"; import type { ActionResult } from "./types"; -const DISCOVERY_WINDOW_INVALID = "DISCOVERY_WINDOW_INVALID"; +function discoveryValidationErrorCode(error: unknown): string | null { + if (!(error instanceof ZodError)) return null; + return getDiscoveryValidationErrorCode(error.issues) ?? null; +} export async function fetchSystemSettings(): Promise> { try { @@ -141,7 +149,7 @@ export async function saveSystemSettings(formData: { return { ok: false, error: "Discovery window validation failed.", - errorCode: DISCOVERY_WINDOW_INVALID, + errorCode: DISCOVERY_WINDOW_INVALID_ERROR_CODE, }; } const updated = await updateSystemSettings({ @@ -280,7 +288,12 @@ export async function saveSystemSettings(formData: { return { ok: true, data: { ...updated, publicStatusProjectionWarningCode } }; } catch (error) { logger.error("更新系统设置失败:", error); - const message = error instanceof Error ? error.message : "更新系统设置失败"; + const validationErrorCode = discoveryValidationErrorCode(error); + const message = validationErrorCode + ? "Discovery settings validation failed." + : error instanceof Error + ? error.message + : "更新系统设置失败"; emitActionAudit({ category: "system_settings", action: "system_settings.update", @@ -290,6 +303,10 @@ export async function saveSystemSettings(formData: { success: false, errorMessage: "UPDATE_FAILED", }); - return { ok: false, error: message }; + return { + ok: false, + error: message, + ...(validationErrorCode ? { errorCode: validationErrorCode } : {}), + }; } } diff --git a/src/app/[locale]/settings/config/_components/system-settings-form.tsx b/src/app/[locale]/settings/config/_components/system-settings-form.tsx index f43859f32..ba8a33b32 100644 --- a/src/app/[locale]/settings/config/_components/system-settings-form.tsx +++ b/src/app/[locale]/settings/config/_components/system-settings-form.tsx @@ -47,6 +47,7 @@ import { shouldWarnQuotaLeaseCapZero, shouldWarnQuotaLeasePercentZero, } from "@/lib/utils/validation/quota-lease-warnings"; +import { DISCOVERY_FIELD_LIMITS } from "@/lib/validation/discovery-settings"; import { DEFAULT_IP_EXTRACTION_CONFIG, type IpExtractionConfig } from "@/types/ip-extraction"; import type { BillingModelSource, @@ -263,12 +264,12 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) racingTotalTimeoutMs: Number(racingTotalTimeoutMs), stickyTimeoutCooldownMs: Number(stickyTimeoutCooldownMs), }; - if ( - discoveryEnabled && - (discoveryConfig.discoveryConcurrency < 2 || - Object.values(discoveryConfig).some((value) => !Number.isSafeInteger(value) || value < 1)) - ) { - toast.error(t("discoveryWindowInvalid")); + const discoverySettingsInvalid = Object.entries(discoveryConfig).some(([field, value]) => { + const [min, max] = DISCOVERY_FIELD_LIMITS[field as keyof typeof DISCOVERY_FIELD_LIMITS]; + return !Number.isSafeInteger(value) || value < min || value > max; + }); + if (discoveryEnabled && discoverySettingsInvalid) { + toast.error(t("discoverySettingsInvalid")); return; } if ( @@ -397,7 +398,9 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps) const errorMessage = result.errorCode === "DISCOVERY_WINDOW_INVALID" ? t("discoveryWindowInvalid") - : result.error || t("saveFailed"); + : result.errorCode === "DISCOVERY_SETTINGS_INVALID" + ? t("discoverySettingsInvalid") + : result.error || t("saveFailed"); toast.error(errorMessage); return; } @@ -725,19 +728,44 @@ export function SystemSettingsForm({ initialSettings }: SystemSettingsFormProps)
{( [ - ["discoveryConcurrency", discoveryConcurrency, setDiscoveryConcurrency, 2], - ["maxDiscoveryRounds", maxDiscoveryRounds, setMaxDiscoveryRounds, 1], - ["discoverySlaMs", discoverySlaMs, setDiscoverySlaMs, 1], - ["stickySlaMs", stickySlaMs, setStickySlaMs, 1], - ["racingTotalTimeoutMs", racingTotalTimeoutMs, setRacingTotalTimeoutMs, 1], + [ + "discoveryConcurrency", + discoveryConcurrency, + setDiscoveryConcurrency, + ...DISCOVERY_FIELD_LIMITS.discoveryConcurrency, + ], + [ + "maxDiscoveryRounds", + maxDiscoveryRounds, + setMaxDiscoveryRounds, + ...DISCOVERY_FIELD_LIMITS.maxDiscoveryRounds, + ], + [ + "discoverySlaMs", + discoverySlaMs, + setDiscoverySlaMs, + ...DISCOVERY_FIELD_LIMITS.discoverySlaMs, + ], + [ + "stickySlaMs", + stickySlaMs, + setStickySlaMs, + ...DISCOVERY_FIELD_LIMITS.stickySlaMs, + ], + [ + "racingTotalTimeoutMs", + racingTotalTimeoutMs, + setRacingTotalTimeoutMs, + ...DISCOVERY_FIELD_LIMITS.racingTotalTimeoutMs, + ], [ "stickyTimeoutCooldownMs", stickyTimeoutCooldownMs, setStickyTimeoutCooldownMs, - 1, + ...DISCOVERY_FIELD_LIMITS.stickyTimeoutCooldownMs, ], ] as const - ).map(([key, value, setter, min]) => ( + ).map(([key, value, setter, min, max]) => (
)} @@ -553,57 +669,137 @@ function TraceValue({ label, value }: { label: string; value: string | number | function AttemptCard({ attempt }: { attempt: AttemptView }) { const t = useTranslations("dashboard.logs.details.routingTrace"); + const [expanded, setExpanded] = useState(false); const style = outcomeStyle(attempt.outcome); const Icon = style.icon; const providerName = attempt.providerName ?? t("providerFallback", { id: attempt.providerId ?? "-" }); + const chainItem = attempt.chainItem; + const statusCode = chainItem?.statusCode ?? attempt.statusCode; + const endpoint = sanitizeEndpoint(chainItem?.endpointUrl); + const errorMessage = getChainErrorMessage(chainItem); + const knownCancellationKinds = new Set([ + "discovery_loser", + "discovery_sla_timeout", + "round_timeout", + "sticky_timeout", + "request_deadline", + "client_abort", + "winner_committed", + ]); + const cancellationLabel = attempt.cancellationKind + ? knownCancellationKinds.has(attempt.cancellationKind) + ? t(`cancellationKinds.${attempt.cancellationKind}`) + : attempt.cancellationKind + : null; return (
-
- -
-
- - {providerName} - - {attempt.elapsedMs != null && ( - - {t("elapsed", { elapsed: Math.max(0, Math.round(attempt.elapsedMs)) })} + + {expanded && ( +
+
+ {attempt.providerId != null && ( + )} - {attempt.statusCode != null && ( - HTTP {attempt.statusCode} + {attempt.sequence != null && ( + + )} + {statusCode != null && ( + )}
- {(attempt.fallbackPromoted || attempt.winnerCommitted) && ( -
- - {attempt.winnerCommitted ? t("winnerCommitted") : t("fallbackPromoted")} + {endpoint && ( +
+
{t("attemptDetails.endpoint")}
+ {endpoint} +
+ )} + {errorMessage && ( +
+
+ {t("attemptDetails.error")} +
+ 
+                {errorMessage}
+
+
+ )} + {cancellationLabel && ( +
+ {t("attemptDetails.cancellation")}:{" "} + {cancellationLabel}
)} {attempt.history.length > 0 && ( -
+
+
{t("attemptDetails.timeline")}
{attempt.history.map((event, index) => (
@@ -619,7 +815,7 @@ function AttemptCard({ attempt }: { attempt: AttemptView }) {
)}
-
+ )}
); } diff --git a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/LogicTraceTab.tsx b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/LogicTraceTab.tsx index 6985debb5..22bc1263e 100644 --- a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/LogicTraceTab.tsx +++ b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/LogicTraceTab.tsx @@ -15,6 +15,7 @@ import { Link2, RefreshCw, Server, + ShieldCheck, XCircle, Zap, } from "lucide-react"; @@ -188,6 +189,9 @@ export function LogicTraceTab({ const totalProviders = decisionContext?.totalProviders || 0; const afterHealthCheck = decisionContext?.afterHealthCheck || 0; const normalizedRoutingTrace = normalizeRoutingTrace(routingTrace); + const isLeaseConflictProtection = + normalizedRoutingTrace?.mode === "single_upstream" && + normalizedRoutingTrace.bypassReason === "lease_conflict"; // Calculate step offset for session reuse flow const sessionReuseStepOffset = isSessionReuseFlow ? 1 : 0; @@ -196,7 +200,7 @@ export function LogicTraceTab({ return (
- +
); } @@ -263,12 +267,16 @@ export function LogicTraceTab({

- {isSessionReuseFlow ? ( + {isLeaseConflictProtection ? ( + + ) : isSessionReuseFlow ? ( ) : ( )} - {t("logicTrace.title")} + {isLeaseConflictProtection + ? t("logicTrace.singleRouteSelectionTitle") + : t("logicTrace.title")}

{isSessionReuseFlow ? ( diff --git a/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.test.tsx b/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.test.tsx index 7906ff832..421936d9f 100644 --- a/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.test.tsx +++ b/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.test.tsx @@ -658,4 +658,88 @@ describe("provider-chain-popover Discovery summary", () => { expect(html).toContain("Note: payload may have been forwarded"); expect(html).toContain("Why CCH cannot retry this response on the server"); }); + + test("marks lease-conflict single-upstream routing without hiding selector priority", () => { + const routingTrace: RoutingTraceV1 = { + version: 1, + mode: "single_upstream", + startedAt: 1_000, + updatedAt: 2_000, + discoveryEnabled: true, + eligible: false, + bypassReason: "lease_conflict", + events: [], + }; + const html = renderWithIntl( + + ); + + expect(html).toContain("Single-route protection"); + expect(html).toContain("Another request owns the Discovery lease"); + expect(html).toContain("P1"); + expect(html).toContain("lucide-shield-check"); + }); + + test("keeps lease-conflict protection visible after serial provider fallback", () => { + const routingTrace: RoutingTraceV1 = { + version: 1, + mode: "single_upstream", + startedAt: 1_000, + updatedAt: 3_000, + discoveryEnabled: true, + eligible: false, + bypassReason: "lease_conflict", + events: [], + }; + const html = renderWithIntl( + + ); + + expect(html).toContain("Single-route protection"); + expect(html).toContain("lucide-shield-check"); + expect(html).toContain("primary"); + expect(html).toContain("backup"); + }); }); diff --git a/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.tsx b/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.tsx index af08d4a4e..0023c669c 100644 --- a/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.tsx +++ b/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.tsx @@ -10,6 +10,7 @@ import { Link2, MinusCircle, RefreshCw, + ShieldCheck, XCircle, Zap, } from "lucide-react"; @@ -331,6 +332,9 @@ export function ProviderChainPopover({ // Determine max width based on whether cost badge is present const maxWidthClass = hasCostBadge ? "max-w-[140px]" : "max-w-[180px]"; + const isLeaseConflictProtection = + normalizedRoutingTrace?.mode === "single_upstream" && + normalizedRoutingTrace.bypassReason === "lease_conflict"; // Check if this is a session reuse const isSessionReuse = @@ -354,22 +358,41 @@ export function ProviderChainPopover({ - + + {isLeaseConflictProtection && ( +
{/* Provider name */}
{displayName}
+ {isLeaseConflictProtection && ( +
+
+ )} {singleRequestItem?.statusCode && (
{/* Request count badge */} - {isHedge ? ( + {isLeaseConflictProtection ? ( +