Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,10 @@ Pre-commit hooks may modify files (e.g. ruff format); re-`git add` if a hook rep
compile arbitrary, sometimes exploit-adjacent code. Sonnet 5.5 declines in more categories than Sonnet 5
(adds `bio`, `reasoning_extraction`, `general_harms`); watch the metric after model bumps. `app/explain.py` returns a distinct user-facing message,
discards any partial output, and emits `ClaudeExplainRefusal`.
- **A spent workspace limit is a 400, not a 429.** Anthropic returns `invalid_request_error` ("You have reached
your specified workspace API usage limits...") until the limit resets or is raised in the Console (a
per-workspace limit, separate from the org one). `app/explain.py` matches the message and emits
`ClaudeExplainUsageLimit`; any other unhandled API error emits `ClaudeExplainApiError` before re-raising.
- **Multi-block responses.** With thinking enabled the API returns thinking blocks before the text block; both
`app/explain.py` and `prompt_testing/runner.py` pick the last text block via
`getattr(c, "type", None) == "text"`. Preserve that pattern in new response-consuming code. "No text block"
Expand Down
37 changes: 33 additions & 4 deletions app/explain.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,18 +90,37 @@ def _transient_error_response(
"Claude Explain took too long to respond — the input may be very large "
"or the model is under heavy load. Please try again in a moment."
)
metric = "ClaudeExplainTimeout"
else:
message_text = "Claude Explain is temporarily unavailable. Please try again in a moment."
metric = "ClaudeExplainTransientError"
LOGGER.warning("Anthropic call failed (%s): %s", type(error).__name__, error)
return _failed_call_response(body, model, metrics_provider, message_text, metric)


def _is_usage_limit_error(error: APIStatusError) -> bool:
"""Whether Anthropic refused the call because the workspace hit its spend limit.

The API reports this as a 400 invalid_request_error rather than a 429, so
without checking the message it looks like a malformed request.
"""
return error.status_code == 400 and "usage limit" in error.message.lower()


def _failed_call_response(
body: ExplainRequest,
model: str,
metrics_provider: MetricsProvider,
message_text: str,
metric: str,
) -> ExplainResponse:
"""Build a structured error response for a Claude call that did not complete."""
metrics_provider.set_property("language", body.language)
metrics_provider.set_property("compiler", body.compiler)
metrics_provider.set_property("instructionSet", body.instructionSet or "unknown")
metrics_provider.set_property("cached", "false")
metrics_provider.put_metric("ClaudeExplainRequest", 1)
if isinstance(error, (TimeoutError, APITimeoutError)):
metrics_provider.put_metric("ClaudeExplainTimeout", 1)
else:
metrics_provider.put_metric("ClaudeExplainTransientError", 1)
metrics_provider.put_metric(metric, 1)
return ExplainResponse(
status="error",
message=message_text,
Expand Down Expand Up @@ -151,10 +170,20 @@ async def _call_anthropic_api(
except (TimeoutError, APITimeoutError, APIConnectionError) as e:
return _transient_error_response(body, prompt_data["model"], metrics_provider, e)
except APIStatusError as e:
if _is_usage_limit_error(e):
LOGGER.error("Anthropic usage limit reached: %s", e)
return _failed_call_response(
body,
prompt_data["model"],
metrics_provider,
"Claude Explain has reached its usage limit and is unavailable for now. Please try again later.",
"ClaudeExplainUsageLimit",
)
# Surface only transient upstream failures gracefully; let genuine
# client errors (e.g. a malformed 400) propagate as a real failure.
if e.status_code in (408, 409, 429, 500, 502, 503, 504, 529):
return _transient_error_response(body, prompt_data["model"], metrics_provider, e)
metrics_provider.put_metric("ClaudeExplainApiError", 1)
raise

# Extract usage information
Expand Down
40 changes: 40 additions & 0 deletions app/test_explain.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock

import httpx
import pytest
from anthropic import BadRequestError
from ruamel.yaml import YAML

from app.explain import process_request
Expand Down Expand Up @@ -359,6 +361,44 @@ async def slow_create(**_kwargs):
assert response.usage is not None
assert response.usage.totalTokens == 0

@staticmethod
def _bad_request(message: str) -> BadRequestError:
response = httpx.Response(400, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"))
body = {"type": "error", "error": {"type": "invalid_request_error", "message": message}}
return BadRequestError(f"Error code: 400 - {body}", response=response, body=body)

@pytest.mark.asyncio
async def test_usage_limit_returns_clear_error(self, sample_request):
"""Anthropic reports a spent workspace limit as a 400; it must become a
structured error with its own metric, not an opaque 500."""
mock_client = MagicMock()
mock_client.messages.create = AsyncMock(
side_effect=self._bad_request(
"You have reached your specified workspace API usage limits. "
"You will regain access on 2026-11-01 at 00:00 UTC."
)
)
metrics = MagicMock()

test_prompt = Prompt(Path("app/prompt.yaml"))
response = await process_request(sample_request, mock_client, test_prompt, metrics)

assert response.status == "error"
assert "usage limit" in response.message
metrics.put_metric.assert_any_call("ClaudeExplainUsageLimit", 1)

@pytest.mark.asyncio
async def test_other_client_errors_still_raise(self, sample_request):
"""A genuine malformed-request 400 still propagates, but is counted so it can alarm."""
mock_client = MagicMock()
mock_client.messages.create = AsyncMock(side_effect=self._bad_request("max_tokens: field required"))
metrics = MagicMock()

test_prompt = Prompt(Path("app/prompt.yaml"))
with pytest.raises(BadRequestError):
await process_request(sample_request, mock_client, test_prompt, metrics)
metrics.put_metric.assert_any_call("ClaudeExplainApiError", 1)


class TestPromptValidation:
"""Validation rules enforced at Prompt construction."""
Expand Down
Loading