fix(proxy): retry upstream storage-capacity 400 failures

src/app/v1/_lib/proxy/errors.ts

@@ -584,6 +584,35 @@ function extractErrorContentForDetection(error: Error): string {
  */
 const errorDetectionCache = new WeakMap<Error, ErrorDetectionResult>();
 
+const RETRYABLE_UPSTREAM_STORAGE_ERROR_MARKERS = [
+  "disk storage creation failed",
+  "disk free-space floor reached",
+] as const;
+
+/**
+ * Detect an upstream storage-capacity failure that was incorrectly returned as HTTP 400.
+ *
+ * This must remain narrow: ordinary invalid-request responses are client errors and should
+ * continue to bypass retries and provider circuit accounting.
+ */
+export function isRetryableUpstreamStorageCapacityError(error: Error): boolean {
+  if (!(error instanceof ProxyError) || error.statusCode !== 400) {
+    return false;
+  }
+
+  // Synthetic fake-200 errors are inferred from a response body and must keep their own
+  // classification even when the body contains the same text as a real upstream response.
+  if (error.message.startsWith("FAKE_200_") || error.upstreamError?.statusCodeInferred === true) {
+    return false;
+  }
+
+  const body = error.upstreamError?.body?.toLowerCase();
+  return (
+    body != null &&
+    RETRYABLE_UPSTREAM_STORAGE_ERROR_MARKERS.every((marker) => body.includes(marker))
+  );
+}
+
 /**
  * 检测错误规则（异步版本，带缓存）
  *
@@ -972,6 +1001,13 @@ export async function categorizeErrorAsync(error: Error): Promise<ErrorCategory>
     return ErrorCategory.SYSTEM_ERROR;
   }
 
+  // Some upstream relays report their own storage-capacity protection as HTTP 400.
+  // Classify this known provider failure before broad client-error rules can match text
+  // such as "invalid request" in the same response body.
+  if (isRetryableUpstreamStorageCapacityError(error)) {
+    return ErrorCategory.PROVIDER_ERROR;
+  }
+
   // 优先级 5: 不可重试的客户端输入错误检测（白名单模式）
   // 使用异步版本确保错误规则已加载
   if (await isNonRetryableClientErrorAsync(error)) {

src/app/v1/_lib/proxy/forwarder.ts

@@ -75,6 +75,7 @@ import {
   isClientAbortError,
   isEmptyResponseError,
   isHttp2Error,
+  isRetryableUpstreamStorageCapacityError,
   isSSLCertificateError,
   ProxyError,
   sanitizeUrl,
@@ -959,6 +960,7 @@ function getReactiveRectifierDisplayName(rectifierType: ReactiveRectifierType):
 }
 
 async function tryApplyReactiveRectifier(params: {
+  error: Error;
   provider: Provider;
   requestSession: ProxySession;
   persistSession: ProxySession;
@@ -967,6 +969,10 @@ async function tryApplyReactiveRectifier(params: {
   retryAttemptNumber: number;
   retryState: ReactiveRectifierRetryState;
 }): Promise<ReactiveRectifierResult> {
+  if (isRetryableUpstreamStorageCapacityError(params.error)) {
+    return { matched: false };
+  }
+
   const {
     provider,
     requestSession,
@@ -1825,6 +1831,7 @@ export class ProxyForwarder {
 
           // 2.5 Reactive rectifier：命中后对同供应商“整流 + 重试一次”
           const reactiveRectifierResult = await tryApplyReactiveRectifier({
+            error: lastError,
             provider: currentProvider,
             requestSession: session,
             persistSession: session,
@@ -4361,6 +4368,7 @@ export class ProxyForwarder {
       }
 
       const reactiveRectifierResult = await tryApplyReactiveRectifier({
+        error,
         provider: attempt.provider,
         requestSession: attempt.session,
         persistSession: session,

tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts

@@ -1874,6 +1874,98 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => {
     }
   });
 
+  test("hedge 路径的上游存储容量型 400 应记录失败并启动替代供应商", async () => {
+    vi.useFakeTimers();
+
+    try {
+      const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 });
+      const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100 });
+      const provider3 = createProvider({ id: 3, name: "p3", firstByteTimeoutStreamingMs: 100 });
+      const session = createSession();
+      session.setProvider(provider1);
+
+      mocks.pickRandomProviderWithExclusion
+        .mockResolvedValueOnce(provider2)
+        .mockResolvedValueOnce(provider3)
+        .mockResolvedValueOnce(null);
+      mocks.categorizeErrorAsync.mockResolvedValue(ProxyErrorCategory.PROVIDER_ERROR);
+
+      const storageError = new UpstreamProxyError(
+        "invalid request: upstream storage failure",
+        400,
+        {
+          body: '{"error":{"message":"disk storage creation failed: failed to write to temp file; disk free-space floor reached"}}',
+          providerId: provider2.id,
+          providerName: provider2.name,
+        }
+      );
+
+      const doForward = vi.spyOn(
+        ProxyForwarder as unknown as {
+          doForward: (...args: unknown[]) => Promise<Response>;
+        },
+        "doForward"
+      );
+      const controller1 = new AbortController();
+      const controller2 = new AbortController();
+      const controller3 = new AbortController();
+
+      doForward.mockImplementationOnce(async (attemptSession) => {
+        const runtime = attemptSession as ProxySession & AttemptRuntime;
+        runtime.responseController = controller1;
+        runtime.clearResponseTimeout = vi.fn();
+        return createStreamingResponse({
+          label: "p1",
+          firstChunkDelayMs: 500,
+          controller: controller1,
+        });
+      });
+
+      doForward.mockImplementationOnce(async (attemptSession) => {
+        const runtime = attemptSession as ProxySession & AttemptRuntime;
+        runtime.responseController = controller2;
+        runtime.clearResponseTimeout = vi.fn();
+        return createDelayedFailure({
+          delayMs: 50,
+          error: storageError,
+          controller: controller2,
+        });
+      });
+
+      doForward.mockImplementationOnce(async (attemptSession) => {
+        const runtime = attemptSession as ProxySession & AttemptRuntime;
+        runtime.responseController = controller3;
+        runtime.clearResponseTimeout = vi.fn();
+        return createStreamingResponse({
+          label: "p3",
+          firstChunkDelayMs: 20,
+          controller: controller3,
+        });
+      });
+
+      const responsePromise = ProxyForwarder.send(session);
+
+      await vi.advanceTimersByTimeAsync(100);
+      expect(doForward).toHaveBeenCalledTimes(2);
+
+      await vi.advanceTimersByTimeAsync(55);
+      expect(doForward).toHaveBeenCalledTimes(3);
+
+      await vi.advanceTimersByTimeAsync(30);
+      const response = await responsePromise;
+
+      expect(await response.text()).toContain('"provider":"p3"');
+      expect(mocks.recordFailure).toHaveBeenCalledWith(provider2.id, storageError);
+      expect(mocks.storeSessionSpecialSettings).not.toHaveBeenCalled();
+      expect(
+        session.getProviderChain().some((entry) => entry.reason === "client_error_non_retryable")
+      ).toBe(false);
+      expect(controller1.signal.aborted).toBe(true);
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
   test("hedge 路径命中 thinking budget 错误时，应整流后在同供应商重试", async () => {
     vi.useFakeTimers();
 

tests/unit/proxy/proxy-forwarder-retry-limit.test.ts

@@ -354,6 +354,69 @@ describe("ProxyForwarder - retry limit enforcement", () => {
     ]);
   });
 
+  test("upstream storage-capacity 400 should retry, record failure, and switch provider", async () => {
+    vi.useFakeTimers();
+
+    try {
+      const session = createSession();
+      const provider1 = createProvider({
+        id: 1,
+        name: "storage-limited-provider",
+        providerVendorId: null,
+        maxRetryAttempts: 2,
+      });
+      const provider2 = createProvider({
+        id: 2,
+        name: "fallback-provider",
+        providerVendorId: null,
+        maxRetryAttempts: 2,
+      });
+      session.setProvider(provider1);
+
+      const storageError = new ProxyError("invalid request: upstream storage failure", 400, {
+        body: '{"error":{"message":"disk storage creation failed: failed to write to temp file; disk free-space floor reached"}}',
+        providerId: provider1.id,
+        providerName: provider1.name,
+      });
+
+      const selectAlternative = vi.spyOn(
+        ProxyForwarder as unknown as {
+          selectAlternative: (...args: unknown[]) => unknown;
+        },
+        "selectAlternative"
+      );
+      selectAlternative.mockResolvedValueOnce(provider2).mockResolvedValueOnce(null);
+
+      const doForward = vi.spyOn(
+        ProxyForwarder as unknown as { doForward: (...args: unknown[]) => unknown },
+        "doForward"
+      );
+      doForward.mockRejectedValueOnce(storageError).mockRejectedValueOnce(storageError);
+      doForward.mockResolvedValueOnce(
+        new Response("{}", {
+          status: 200,
+          headers: { "content-type": "application/json", "content-length": "2" },
+        })
+      );
+
+      const sendPromise = ProxyForwarder.send(session);
+      await vi.runAllTimersAsync();
+      const response = await sendPromise;
+
+      expect(response.status).toBe(200);
+      expect(doForward).toHaveBeenCalledTimes(3);
+      expect(selectAlternative).toHaveBeenCalledWith(session, [provider1.id]);
+      expect(mocks.recordFailure).toHaveBeenCalledTimes(1);
+      expect(mocks.recordFailure).toHaveBeenCalledWith(provider1.id, storageError);
+      expect(session.provider?.id).toBe(provider2.id);
+      expect(
+        session.getProviderChain().some((entry) => entry.reason === "client_error_non_retryable")
+      ).toBe(false);
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
   test("endpoints > maxRetry: should only use top N lowest-latency endpoints", async () => {
     vi.useFakeTimers();
 

tests/unit/proxy/upstream-storage-capacity-error.test.ts

@@ -0,0 +1,93 @@
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("@/repository/error-rules", () => ({
+  getActiveErrorRules: vi.fn(async () => [
+    {
+      id: 29,
+      pattern: "非法请求|illegal request|invalid request",
+      matchType: "regex",
+      category: "invalid_request",
+      description: "Invalid request format",
+      overrideResponse: null,
+      overrideStatusCode: null,
+      isEnabled: true,
+      isDefault: true,
+      priority: 50,
+      createdAt: new Date(0),
+      updatedAt: new Date(0),
+    },
+  ]),
+}));
+
+import {
+  categorizeErrorAsync,
+  ErrorCategory,
+  isRetryableUpstreamStorageCapacityError,
+  ProxyError,
+} from "@/app/v1/_lib/proxy/errors";
+
+const STORAGE_ERROR_BODY =
+  '{"error":{"type":"invalid_request_error","message":"disk storage creation failed: failed to write to temp file; disk free-space floor reached"}}';
+
+function createStorageError(
+  body: string = STORAGE_ERROR_BODY,
+  upstreamError: Record<string, unknown> = {},
+  message = "invalid request: upstream storage failure"
+): ProxyError {
+  return new ProxyError(message, 400, {
+    body,
+    providerId: 80,
+    providerName: "test-provider",
+    ...upstreamError,
+  });
+}
+
+describe("upstream storage-capacity 400 classification", () => {
+  it("prioritizes the known provider failure over the broad invalid-request rule", async () => {
+    const error = createStorageError();
+
+    expect(isRetryableUpstreamStorageCapacityError(error)).toBe(true);
+    expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.PROVIDER_ERROR);
+  });
+
+  it("keeps ordinary invalid-request 400 responses non-retryable", async () => {
+    const error = createStorageError('{"error":{"message":"invalid request: malformed JSON"}}');
+
+    expect(isRetryableUpstreamStorageCapacityError(error)).toBe(false);
+    expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR);
+  });
+
+  it("requires both stable storage markers instead of retrying every 400 with one marker", async () => {
+    const error = createStorageError(
+      '{"error":{"message":"invalid request: disk storage creation failed"}}'
+    );
+
+    expect(isRetryableUpstreamStorageCapacityError(error)).toBe(false);
+    expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR);
+  });
+
+  it("excludes a fake-200 message prefix without relying on inferred status", async () => {
+    const error = createStorageError(
+      STORAGE_ERROR_BODY.replace("disk storage", "invalid request: disk storage"),
+      {
+        statusCodeInferred: false,
+      },
+      "FAKE_200_JSON_ERROR_NON_EMPTY"
+    );
+
+    expect(isRetryableUpstreamStorageCapacityError(error)).toBe(false);
+    expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR);
+  });
+
+  it("excludes an inferred status without relying on a fake-200 message prefix", async () => {
+    const error = createStorageError(
+      STORAGE_ERROR_BODY.replace("disk storage", "invalid request: disk storage"),
+      {
+        statusCodeInferred: true,
+      }
+    );
+
+    expect(isRetryableUpstreamStorageCapacityError(error)).toBe(false);
+    expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR);
+  });
+});