fix(proxy): fail over provider-local model 404s

src/app/v1/_lib/proxy/errors.ts

@@ -576,6 +576,24 @@ function extractErrorContentForDetection(error: Error): string {
   return error.message;
 }
 
+const PROVIDER_LOCAL_MODEL_UNAVAILABLE_MARKER =
+  "not supported by any configured account in this group";
+
+/** A model capability gap local to one upstream account pool, not to the request. */
+export function isProviderLocalModelUnavailableError(error: unknown): error is ProxyError {
+  if (
+    !(error instanceof ProxyError) ||
+    error.statusCode !== 404 ||
+    error.upstreamError?.statusCodeInferred === true
+  ) {
+    return false;
+  }
+
+  return [error.message, error.upstreamError?.body].some((content) =>
+    content?.toLowerCase().includes(PROVIDER_LOCAL_MODEL_UNAVAILABLE_MARKER)
+  );
+}
+
 /**
  * 错误规则检测结果缓存
  *
@@ -1001,6 +1019,13 @@ export async function categorizeErrorAsync(error: Error): Promise<ErrorCategory>
     return ErrorCategory.SYSTEM_ERROR;
   }
 
+  // Some upstream account pools use model_not_found for a provider-local capability
+  // gap. The request may still succeed on another Provider, so classify this exact
+  // 404 before broad client-input rules match the generic model_not_found wording.
+  if (isProviderLocalModelUnavailableError(error)) {
+    return ErrorCategory.RESOURCE_NOT_FOUND;
+  }
+
   // Some upstream relays report their own storage-capacity protection as HTTP 400.
   // Classify this known provider failure before broad client-error rules can match text
   // such as "invalid request" in the same response body.

src/app/v1/_lib/proxy/forwarder.ts

@@ -75,6 +75,7 @@ import {
   isClientAbortError,
   isEmptyResponseError,
   isHttp2Error,
+  isProviderLocalModelUnavailableError,
   isRetryableUpstreamStorageCapacityError,
   isSSLCertificateError,
   ProxyError,
@@ -969,7 +970,10 @@ async function tryApplyReactiveRectifier(params: {
   retryAttemptNumber: number;
   retryState: ReactiveRectifierRetryState;
 }): Promise<ReactiveRectifierResult> {
-  if (isRetryableUpstreamStorageCapacityError(params.error)) {
+  if (
+    isProviderLocalModelUnavailableError(params.error) ||
+    isRetryableUpstreamStorageCapacityError(params.error)
+  ) {
     return { matched: false };
   }
 
@@ -2054,10 +2058,12 @@ export class ProxyForwarder {
             break; // ⭐ 跳出内层循环，进入供应商切换逻辑
           }
 
-          // ⭐ 5. 上游 404 错误处理（不计入熔断器，先重试当前供应商，重试耗尽后切换）
+          // 5. 上游 404 错误处理（不计入熔断器；Provider 局部模型缺口直接切换）
           if (errorCategory === ErrorCategory.RESOURCE_NOT_FOUND) {
             const proxyError = lastError as ProxyError;
-            const willRetry = attemptCount < maxAttemptsPerProvider;
+            const providerLocalModelUnavailable = isProviderLocalModelUnavailableError(proxyError);
+            const willRetry =
+              !providerLocalModelUnavailable && attemptCount < maxAttemptsPerProvider;
 
             logger.warn("ProxyForwarder: Upstream 404 error", {
               providerId: currentProvider.id,
@@ -2068,6 +2074,7 @@ export class ProxyForwarder {
               attemptNumber: attemptCount,
               totalProvidersAttempted,
               willRetry,
+              providerLocalModelUnavailable,
             });
 
             // 记录到决策链（标记为 resource_not_found，不计入熔断）

tests/unit/proxy/provider-local-model-unavailable.test.ts

@@ -0,0 +1,135 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  ErrorCategory,
+  ProxyError,
+  categorizeErrorAsync,
+  isProviderLocalModelUnavailableError,
+} from "@/app/v1/_lib/proxy/errors";
+import type { ErrorDetectionResult } from "@/lib/error-rule-detector";
+
+const mocks = vi.hoisted(() => ({
+  detectAsync: vi.fn<(content: string) => Promise<ErrorDetectionResult>>(),
+}));
+
+vi.mock("@/lib/error-rule-detector", () => ({
+  errorRuleDetector: { detectAsync: mocks.detectAsync },
+}));
+
+describe("provider-local model availability errors", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mocks.detectAsync.mockResolvedValue({ matched: true });
+  });
+
+  it("switches Provider when an account group does not support the requested model", async () => {
+    const body = JSON.stringify({
+      error: {
+        message: 'Model "gpt-5.6-sol" is not supported by any configured account in this group',
+        type: "model_not_found",
+      },
+    });
+    const response = new Response(body, {
+      status: 404,
+      headers: { "content-type": "application/json" },
+    });
+    const error = await ProxyError.fromUpstreamResponse(response, {
+      id: 96,
+      name: "provider-a",
+    });
+
+    expect(isProviderLocalModelUnavailableError(error)).toBe(true);
+    expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.RESOURCE_NOT_FOUND);
+    expect(mocks.detectAsync).not.toHaveBeenCalled();
+  });
+
+  it("matches the provider-local marker case-insensitively in the extracted message", async () => {
+    const error = new ProxyError(
+      "MODEL IS NOT SUPPORTED BY ANY CONFIGURED ACCOUNT IN THIS GROUP",
+      404,
+      {
+        body: "{}",
+        providerId: 96,
+        providerName: "provider-a",
+      }
+    );
+
+    expect(isProviderLocalModelUnavailableError(error)).toBe(true);
+    expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.RESOURCE_NOT_FOUND);
+    expect(mocks.detectAsync).not.toHaveBeenCalled();
+  });
+
+  it("does not use the raw upstream body for provider-local matching", async () => {
+    const error = new ProxyError("model_not_found", 404, {
+      body: "{}",
+      rawBody: "not supported by any configured account in this group",
+      providerId: 96,
+      providerName: "provider-a",
+    });
+
+    expect(isProviderLocalModelUnavailableError(error)).toBe(false);
+    expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR);
+    expect(mocks.detectAsync).toHaveBeenCalledOnce();
+  });
+
+  it("keeps ordinary model_not_found responses non-retryable", async () => {
+    const error = new ProxyError("The requested model was not found", 404, {
+      body: JSON.stringify({
+        error: {
+          code: "model_not_found",
+          message: "The requested model was not found",
+        },
+      }),
+      providerId: 96,
+      providerName: "provider-a",
+    });
+
+    expect(isProviderLocalModelUnavailableError(error)).toBe(false);
+    expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR);
+    expect(mocks.detectAsync).toHaveBeenCalledOnce();
+  });
+
+  it("does not match a near-miss account-group message", async () => {
+    const error = new ProxyError(
+      "Model is not supported by every configured account in this group",
+      404,
+      {
+        body: '{"error":"model_not_found"}',
+        providerId: 96,
+        providerName: "provider-a",
+      }
+    );
+
+    expect(isProviderLocalModelUnavailableError(error)).toBe(false);
+    expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR);
+    expect(mocks.detectAsync).toHaveBeenCalledOnce();
+  });
+
+  it("does not override the same wording on a non-404 client error", async () => {
+    const error = new ProxyError(
+      "Model is not supported by any configured account in this group",
+      400,
+      {
+        body: "invalid request",
+        providerId: 96,
+        providerName: "provider-a",
+      }
+    );
+
+    expect(isProviderLocalModelUnavailableError(error)).toBe(false);
+    expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR);
+    expect(mocks.detectAsync).toHaveBeenCalledOnce();
+  });
+
+  it("does not override a synthetic 404 inferred from a fake-200 response", async () => {
+    const error = new ProxyError("not supported by any configured account in this group", 404, {
+      body: '{"error":"model_not_found"}',
+      statusCodeInferred: true,
+      providerId: 96,
+      providerName: "provider-a",
+    });
+
+    expect(isProviderLocalModelUnavailableError(error)).toBe(false);
+    expect(await categorizeErrorAsync(error)).toBe(ErrorCategory.NON_RETRYABLE_CLIENT_ERROR);
+    expect(mocks.detectAsync).toHaveBeenCalledOnce();
+  });
+});

tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts

@@ -1558,6 +1558,82 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => {
     }
   });
 
+  test("provider-local model 404 should not cancel another in-flight hedge candidate", async () => {
+    vi.useFakeTimers();
+
+    try {
+      const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 });
+      const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100 });
+      const session = createSession();
+      session.setProvider(provider1);
+
+      mocks.pickRandomProviderWithExclusion
+        .mockResolvedValueOnce(provider2)
+        .mockResolvedValueOnce(null);
+      mocks.categorizeErrorAsync.mockResolvedValue(ProxyErrorCategory.RESOURCE_NOT_FOUND);
+
+      const doForward = vi.spyOn(
+        ProxyForwarder as unknown as {
+          doForward: (...args: unknown[]) => Promise<Response>;
+        },
+        "doForward"
+      );
+      const controller1 = new AbortController();
+      const controller2 = new AbortController();
+      const providerLocal404 = new UpstreamProxyError(
+        'Model "gpt-5.6-sol" is not supported by any configured account in this group',
+        404,
+        {
+          body: '{"error":{"type":"model_not_found","message":"invalid request: not supported by any configured account in this group"}}',
+          providerId: provider1.id,
+          providerName: provider1.name,
+        }
+      );
+
+      doForward.mockImplementationOnce(async (attemptSession) => {
+        const runtime = attemptSession as ProxySession & AttemptRuntime;
+        runtime.responseController = controller1;
+        runtime.clearResponseTimeout = vi.fn();
+        return createDelayedFailure({
+          delayMs: 150,
+          error: providerLocal404,
+          controller: controller1,
+        });
+      });
+      doForward.mockImplementationOnce(async (attemptSession) => {
+        const runtime = attemptSession as ProxySession & AttemptRuntime;
+        runtime.responseController = controller2;
+        runtime.clearResponseTimeout = vi.fn();
+        return createStreamingResponse({
+          label: "p2",
+          firstChunkDelayMs: 80,
+          controller: controller2,
+        });
+      });
+
+      const responsePromise = ProxyForwarder.send(session);
+
+      await vi.advanceTimersByTimeAsync(100);
+      expect(doForward).toHaveBeenCalledTimes(2);
+      await vi.advanceTimersByTimeAsync(50);
+      expect(controller2.signal.aborted).toBe(false);
+      await vi.advanceTimersByTimeAsync(30);
+
+      const response = await responsePromise;
+      expect(await response.text()).toContain('"provider":"p2"');
+      expect(controller2.signal.aborted).toBe(false);
+      expect(mocks.recordFailure).not.toHaveBeenCalledWith(provider1.id, providerLocal404);
+      expect(session.getProviderChain()).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({ id: provider1.id, reason: "resource_not_found" }),
+          expect.objectContaining({ id: provider2.id, reason: "hedge_winner" }),
+        ])
+      );
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
   test.each([
     {
       name: "provider error",
@@ -1570,14 +1646,18 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => {
         }),
     },
     {
-      name: "resource not found",
+      name: "provider-local resource not found",
       category: ProxyErrorCategory.RESOURCE_NOT_FOUND,
       errorFactory: (provider: Provider) =>
-        new UpstreamProxyError("Provider returned 404: model not found", 404, {
-          body: '{"error":"model_not_found"}',
-          providerId: provider.id,
-          providerName: provider.name,
-        }),
+        new UpstreamProxyError(
+          'Model "gpt-5.6-sol" is not supported by any configured account in this group',
+          404,
+          {
+            body: '{"error":{"type":"model_not_found"}}',
+            providerId: provider.id,
+            providerName: provider.name,
+          }
+        ),
     },
     {
       name: "system error",

tests/unit/proxy/proxy-forwarder-retry-limit.test.ts

@@ -354,6 +354,67 @@ describe("ProxyForwarder - retry limit enforcement", () => {
     ]);
   });
 
+  test("provider-local model 404 skips same-provider retries and switches Provider", async () => {
+    const session = createSession();
+    const provider1 = createProvider({
+      id: 1,
+      name: "provider-without-model",
+      providerVendorId: null,
+      maxRetryAttempts: 3,
+    });
+    const provider2 = createProvider({
+      id: 2,
+      name: "provider-with-model",
+      providerVendorId: null,
+    });
+    session.setProvider(provider1);
+
+    mocks.getPreferredProviderEndpoints.mockResolvedValue([]);
+    vi.mocked(categorizeErrorAsync).mockResolvedValue(ErrorCategory.RESOURCE_NOT_FOUND);
+
+    const doForward = vi.spyOn(
+      ProxyForwarder as unknown as { doForward: (...args: unknown[]) => unknown },
+      "doForward"
+    );
+    const selectAlternative = vi.spyOn(
+      ProxyForwarder as unknown as { selectAlternative: (...args: unknown[]) => unknown },
+      "selectAlternative"
+    );
+    const providerLocal404 = new ProxyError(
+      'Model "gpt-5.6-sol" is not supported by any configured account in this group',
+      404,
+      {
+        body: '{"error":{"type":"model_not_found","message":"invalid request: not supported by any configured account in this group"}}',
+        providerId: provider1.id,
+        providerName: provider1.name,
+      }
+    );
+
+    doForward.mockRejectedValueOnce(providerLocal404).mockResolvedValueOnce(
+      new Response("{}", {
+        status: 200,
+        headers: { "content-type": "application/json", "content-length": "2" },
+      })
+    );
+    selectAlternative.mockResolvedValueOnce(provider2);
+
+    const response = await ProxyForwarder.send(session);
+
+    expect(response.status).toBe(200);
+    expect(doForward).toHaveBeenCalledTimes(2);
+    expect(selectAlternative).toHaveBeenCalledTimes(1);
+    expect(selectAlternative).toHaveBeenCalledWith(session, [provider1.id]);
+    expect(mocks.recordFailure).not.toHaveBeenCalled();
+    expect(session.getProviderChain()).toEqual([
+      expect.objectContaining({
+        id: provider1.id,
+        reason: "resource_not_found",
+        attemptNumber: 1,
+      }),
+      expect.objectContaining({ id: provider2.id, reason: "retry_success", attemptNumber: 1 }),
+    ]);
+  });
+
   test("upstream storage-capacity 400 should retry, record failure, and switch provider", async () => {
     vi.useFakeTimers();