diff --git a/src/app/v1/_lib/proxy/errors.ts b/src/app/v1/_lib/proxy/errors.ts index 2c9d82419..4581a78c8 100644 --- a/src/app/v1/_lib/proxy/errors.ts +++ b/src/app/v1/_lib/proxy/errors.ts @@ -584,6 +584,35 @@ function extractErrorContentForDetection(error: Error): string { */ const errorDetectionCache = new WeakMap(); +const RETRYABLE_UPSTREAM_STORAGE_ERROR_MARKERS = [ + "disk storage creation failed", + "disk free-space floor reached", +] as const; + +/** + * Detect an upstream storage-capacity failure that was incorrectly returned as HTTP 400. + * + * This must remain narrow: ordinary invalid-request responses are client errors and should + * continue to bypass retries and provider circuit accounting. + */ +export function isRetryableUpstreamStorageCapacityError(error: Error): boolean { + if (!(error instanceof ProxyError) || error.statusCode !== 400) { + return false; + } + + // Synthetic fake-200 errors are inferred from a response body and must keep their own + // classification even when the body contains the same text as a real upstream response. + if (error.message.startsWith("FAKE_200_") || error.upstreamError?.statusCodeInferred === true) { + return false; + } + + const body = error.upstreamError?.body?.toLowerCase(); + return ( + body != null && + RETRYABLE_UPSTREAM_STORAGE_ERROR_MARKERS.every((marker) => body.includes(marker)) + ); +} + /** * 检测错误规则（异步版本，带缓存） * @@ -972,6 +1001,13 @@ export async function categorizeErrorAsync(error: Error): Promise return ErrorCategory.SYSTEM_ERROR; } + // Some upstream relays report their own storage-capacity protection as HTTP 400. + // Classify this known provider failure before broad client-error rules can match text + // such as "invalid request" in the same response body. + if (isRetryableUpstreamStorageCapacityError(error)) { + return ErrorCategory.PROVIDER_ERROR; + } + // 优先级 5: 不可重试的客户端输入错误检测（白名单模式） // 使用异步版本确保错误规则已加载 if (await isNonRetryableClientErrorAsync(error)) { diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index 966afc6a7..e389be940 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -75,6 +75,7 @@ import { isClientAbortError, isEmptyResponseError, isHttp2Error, + isRetryableUpstreamStorageCapacityError, isSSLCertificateError, ProxyError, sanitizeUrl, @@ -959,6 +960,7 @@ function getReactiveRectifierDisplayName(rectifierType: ReactiveRectifierType): } async function tryApplyReactiveRectifier(params: { + error: Error; provider: Provider; requestSession: ProxySession; persistSession: ProxySession; @@ -967,6 +969,10 @@ async function tryApplyReactiveRectifier(params: { retryAttemptNumber: number; retryState: ReactiveRectifierRetryState; }): Promise { + if (isRetryableUpstreamStorageCapacityError(params.error)) { + return { matched: false }; + } + const { provider, requestSession, @@ -1825,6 +1831,7 @@ export class ProxyForwarder { // 2.5 Reactive rectifier：命中后对同供应商“整流 + 重试一次” const reactiveRectifierResult = await tryApplyReactiveRectifier({ + error: lastError, provider: currentProvider, requestSession: session, persistSession: session, @@ -4361,6 +4368,7 @@ export class ProxyForwarder { } const reactiveRectifierResult = await tryApplyReactiveRectifier({ + error, provider: attempt.provider, requestSession: attempt.session, persistSession: session, diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index 014219a24..835dd4eff 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -1874,6 +1874,98 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { } }); + test("hedge 路径的上游存储容量型 400 应记录失败并启动替代供应商", async () => { + vi.useFakeTimers(); + + try { + const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); + const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100 }); + const provider3 = createProvider({ id: 3, name: "p3", firstByteTimeoutStreamingMs: 100 }); + const session = createSession(); + session.setProvider(provider1); + + mocks.pickRandomProviderWithExclusion + .mockResolvedValueOnce(provider2) + .mockResolvedValueOnce(provider3) + .mockResolvedValueOnce(null); + mocks.categorizeErrorAsync.mockResolvedValue(ProxyErrorCategory.PROVIDER_ERROR); + + const storageError = new UpstreamProxyError( + "invalid request: upstream storage failure", + 400, + { + body: '{"error":{"message":"disk storage creation failed: failed to write to temp file; disk free-space floor reached"}}', + providerId: provider2.id, + providerName: provider2.name, + } + ); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + const controller1 = new AbortController(); + const controller2 = new AbortController(); + const controller3 = new AbortController(); + + doForward.mockImplementationOnce(async (attemptSession) => { + const runtime = attemptSession as ProxySession & AttemptRuntime; + runtime.responseController = controller1; + runtime.clearResponseTimeout = vi.fn(); + return createStreamingResponse({ + label: "p1", + firstChunkDelayMs: 500, + controller: controller1, + }); + }); + + doForward.mockImplementationOnce(async (attemptSession) => { + const runtime = attemptSession as ProxySession & AttemptRuntime; + runtime.responseController = controller2; + runtime.clearResponseTimeout = vi.fn(); + return createDelayedFailure({ + delayMs: 50, + error: storageError, + controller: controller2, + }); + }); + + doForward.mockImplementationOnce(async (attemptSession) => { + const runtime = attemptSession as ProxySession & AttemptRuntime; + runtime.responseController = controller3; + runtime.clearResponseTimeout = vi.fn(); + return createStreamingResponse({ + label: "p3", + firstChunkDelayMs: 20, + controller: controller3, + }); + }); + + const responsePromise = ProxyForwarder.send(session); + + await vi.advanceTimersByTimeAsync(100); + expect(doForward).toHaveBeenCalledTimes(2); + + await vi.advanceTimersByTimeAsync(55); + expect(doForward).toHaveBeenCalledTimes(3); + + await vi.advanceTimersByTimeAsync(30); + const response = await responsePromise; + + expect(await response.text()).toContain('"provider":"p3"'); + expect(mocks.recordFailure).toHaveBeenCalledWith(provider2.id, storageError); + expect(mocks.storeSessionSpecialSettings).not.toHaveBeenCalled(); + expect( + session.getProviderChain().some((entry) => entry.reason === "client_error_non_retryable") + ).toBe(false); + expect(controller1.signal.aborted).toBe(true); + } finally { + vi.useRealTimers(); + } + }); + test("hedge 路径命中 thinking budget 错误时，应整流后在同供应商重试", async () => { vi.useFakeTimers(); diff --git a/tests/unit/proxy/proxy-forwarder-retry-limit.test.ts b/tests/unit/proxy/proxy-forwarder-retry-limit.test.ts index 454b3a7d6..5f99a68a2 100644 --- a/tests/unit/proxy/proxy-forwarder-retry-limit.test.ts +++ b/tests/unit/proxy/proxy-forwarder-retry-limit.test.ts @@ -354,6 +354,69 @@ describe("ProxyForwarder - retry limit enforcement", () => { ]); }); + test("upstream storage-capacity 400 should retry, record failure, and switch provider", async () => { + vi.useFakeTimers(); + + try { + const session = createSession(); + const provider1 = createProvider({ + id: 1, + name: "storage-limited-provider", + providerVendorId: null, + maxRetryAttempts: 2, + }); + const provider2 = createProvider({ + id: 2, + name: "fallback-provider", + providerVendorId: null, + maxRetryAttempts: 2, + }); + session.setProvider(provider1); + + const storageError = new ProxyError("invalid request: upstream storage failure", 400, { + body: '{"error":{"message":"disk storage creation failed: failed to write to temp file; disk free-space floor reached"}}', + providerId: provider1.id, + providerName: provider1.name, + }); + + const selectAlternative = vi.spyOn( + ProxyForwarder as unknown as { + selectAlternative: (...args: unknown[]) => unknown; + }, + "selectAlternative" + ); + selectAlternative.mockResolvedValueOnce(provider2).mockResolvedValueOnce(null); + + const doForward = vi.spyOn( + ProxyForwarder as unknown as { doForward: (...args: unknown[]) => unknown }, + "doForward" + ); + doForward.mockRejectedValueOnce(storageError).mockRejectedValueOnce(storageError); + doForward.mockResolvedValueOnce( + new Response("{}", { + status: 200, + headers: { "content-type": "application/json", "content-length": "2" }, + }) + ); + + const sendPromise = ProxyForwarder.send(session); + await vi.runAllTimersAsync(); + const response = await sendPromise; + + expect(response.status).toBe(200); + expect(doForward).toHaveBeenCalledTimes(3); + expect(selectAlternative).toHaveBeenCalledWith(session, [provider1.id]); + expect(mocks.recordFailure).toHaveBeenCalledTimes(1); + expect(mocks.recordFailure).toHaveBeenCalledWith(provider1.id, storageError); + expect(session.provider?.id).toBe(provider2.id); + expect( + session.getProviderChain().some((entry) => entry.reason === "client_error_non_retryable") + ).toBe(false); + } finally { + vi.useRealTimers(); + } + }); + test("endpoints > maxRetry: should only use top N lowest-latency endpoints", async () => { vi.useFakeTimers(); diff --git a/tests/unit/proxy/upstream-storage-capacity-error.test.ts b/tests/unit/proxy/upstream-storage-capacity-error.test.ts new file mode 100644 index 000000000..94b147157 --- /dev/null +++ b/tests/unit/proxy/upstream-storage-capacity-error.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/repository/error-rules", () => ({ + getActiveErrorRules: vi.fn(async () => [ + { + id: 29, + pattern: "非法请求|illegal request|invalid request", + matchType: "regex", + category: "invalid_request", + description: "Invalid request format", + overrideResponse: null, + overrideStatusCode: null, + isEnabled: true, + isDefault: true, + priority: 50, + createdAt: new Date(0), + updatedAt: new Date(0), + }, + ]), +})); + +import { + categorizeErrorAsync, + ErrorCategory, + isRetryableUpstreamStorageCapacityError, + ProxyError, +} from "@/app/v1/_lib/proxy/errors"; + +const STORAGE_ERROR_BODY = + '{"error":{"type":"invalid_request_error","message":"disk storage creation failed: failed to write to temp file; disk free-space floor reached"}}'; + +function createStorageError( + body: string = STORAGE_ERROR_BODY, + upstreamError: Record = {}, + message = "invalid request: upstream storage failure" +): ProxyError { + return new ProxyError(message, 400, { + body, + providerId: 80, + providerName: "test-provider", + ...upstreamError, + }); +} + +describe("upstream storage-capacity 400 classification", () => { + it("prioritizes the known provider failure over the broad invalid-request rule", async () => { + const error = createStorageError(); + + expect(isRetryableUpstreamStorageCapacityError(error)).toBe(true); + expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.PROVIDER_ERROR); + }); + + it("keeps ordinary invalid-request 400 responses non-retryable", async () => { + const error = createStorageError('{"error":{"message":"invalid request: malformed JSON"}}'); + + expect(isRetryableUpstreamStorageCapacityError(error)).toBe(false); + expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR); + }); + + it("requires both stable storage markers instead of retrying every 400 with one marker", async () => { + const error = createStorageError( + '{"error":{"message":"invalid request: disk storage creation failed"}}' + ); + + expect(isRetryableUpstreamStorageCapacityError(error)).toBe(false); + expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR); + }); + + it("excludes a fake-200 message prefix without relying on inferred status", async () => { + const error = createStorageError( + STORAGE_ERROR_BODY.replace("disk storage", "invalid request: disk storage"), + { + statusCodeInferred: false, + }, + "FAKE_200_JSON_ERROR_NON_EMPTY" + ); + + expect(isRetryableUpstreamStorageCapacityError(error)).toBe(false); + expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR); + }); + + it("excludes an inferred status without relying on a fake-200 message prefix", async () => { + const error = createStorageError( + STORAGE_ERROR_BODY.replace("disk storage", "invalid request: disk storage"), + { + statusCodeInferred: true, + } + ); + + expect(isRetryableUpstreamStorageCapacityError(error)).toBe(false); + expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR); + }); +});