From fec4ae69e0eb5ac8f18b3c2845d6a207ef4bb65e Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 16 May 2026 02:46:47 +0000 Subject: [PATCH 01/10] test(ci): add reasoning_effort grid v4 e2e regression suite Encode the 231-cell QA sweep (21 provider x model combos x 11 effort values) from #27039 / #27074 as an automated CircleCI-gated regression suite. Each cell hits the real provider endpoint, captures the outgoing wire body via a pre-call CustomLogger, and asserts: - thinking.type, output_config.effort, thinking.budget_tokens, max_tokens in the captured request body (regression signal for silent drops/strips in any provider transformation) - HTTP status (200 vs BadRequestError -> 400) returned by litellm (regression signal for clean-error vs leaked-500 mappings) The matrix is encoded as a small rule set keyed by (model_mode, effort) plus per-model xhigh/max capability overrides, then expanded across the five chat-completion routes (Anthropic direct, Azure AI Foundry, Vertex AI, Bedrock Converse, Bedrock Invoke /chat) and the Bedrock Invoke /v1/messages route. Cells skip at runtime when the route's provider env vars are absent, so PR builds without credentials no-op gracefully. Wired into CircleCI as the reasoning_effort_grid_v4_e2e job behind the existing main / litellm_* branch filter. --- .circleci/config.yml | 44 +++ .../reasoning_effort_grid_v4/__init__.py | 0 .../reasoning_effort_grid_v4/conftest.py | 61 ++++ .../reasoning_effort_grid_v4/grid_spec.py | 301 ++++++++++++++++++ .../reasoning_effort_grid_v4/test_grid_v4.py | 230 +++++++++++++ 5 files changed, 636 insertions(+) create mode 100644 tests/test_litellm/reasoning_effort_grid_v4/__init__.py create mode 100644 tests/test_litellm/reasoning_effort_grid_v4/conftest.py create mode 100644 tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py create mode 100644 tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a5b6da77f..3dd1221c12 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -576,6 +576,48 @@ jobs: no_output_timeout: 15m # Store test results + - store_test_results: + path: test-results + reasoning_effort_grid_v4_e2e: + docker: + - *python312_image + working_directory: ~/project + resource_class: large + + steps: + - checkout + - setup_google_dns + - install_uv + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - save_cache: + paths: + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} + # Grid v4 exercises reasoning_effort mapping against real Anthropic, + # Azure AI Foundry, Vertex AI, Bedrock Converse, and Bedrock Invoke + # endpoints. Per-route cells pytest-skip themselves when the matching + # provider env vars are absent, so PRs without credentials no-op. + - run: + name: Run reasoning_effort grid v4 e2e suite + command: | + mkdir -p test-results + uv run --no-sync python -m pytest \ + tests/test_litellm/reasoning_effort_grid_v4/ \ + -v \ + --junitxml=test-results/junit.xml \ + --durations=20 \ + -n 4 \ + --timeout=180 --timeout_method=thread \ + --retries 2 --retry-delay 5 \ + --max-worker-restart=5 + no_output_timeout: 20m + - store_test_results: path: test-results realtime_translation_testing: @@ -2619,6 +2661,8 @@ workflows: filters: *main_branches - llm_translation_testing: filters: *main_branches + - reasoning_effort_grid_v4_e2e: + filters: *main_branches - realtime_translation_testing: filters: *main_branches - agent_testing: diff --git a/tests/test_litellm/reasoning_effort_grid_v4/__init__.py b/tests/test_litellm/reasoning_effort_grid_v4/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_litellm/reasoning_effort_grid_v4/conftest.py b/tests/test_litellm/reasoning_effort_grid_v4/conftest.py new file mode 100644 index 0000000000..5e1a4e0e97 --- /dev/null +++ b/tests/test_litellm/reasoning_effort_grid_v4/conftest.py @@ -0,0 +1,61 @@ +"""Shared fixtures for the reasoning_effort grid v4 e2e suite.""" + +import os +from typing import Any, Dict, List, Optional + +import pytest + +import litellm +from litellm.integrations.custom_logger import CustomLogger + + +class _WireBodyCapture(CustomLogger): + """Pre-call hook that records the outgoing wire body LiteLLM sends upstream. + + `complete_input_dict` is the fully transformed provider request as set by + every provider transformation in `litellm/llms/**`. Capturing it here means + a regression anywhere in the transformation chain (strip, rename, drop) + surfaces as an assertion failure on the cell that depends on it. + """ + + def __init__(self) -> None: + super().__init__() + self.records: List[Dict[str, Any]] = [] + + def log_pre_api_call(self, model, messages, kwargs): + self.records.append( + { + "model": model, + "body": kwargs.get("additional_args", {}).get("complete_input_dict"), + "api_base": kwargs.get("additional_args", {}).get("api_base"), + } + ) + + async def async_log_pre_api_call(self, model, messages, kwargs): + self.log_pre_api_call(model, messages, kwargs) + + def latest(self) -> Optional[Dict[str, Any]]: + return self.records[-1] if self.records else None + + def reset(self) -> None: + self.records.clear() + + +@pytest.fixture() +def wire_capture(): + capture = _WireBodyCapture() + previous = list(litellm.callbacks) + litellm.callbacks = previous + [capture] + try: + yield capture + finally: + litellm.callbacks = previous + + +@pytest.fixture(scope="session") +def vertex_credentials_path() -> Optional[str]: + """Resolve a usable Vertex credentials file path or None.""" + path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + if path and os.path.exists(path): + return path + return None diff --git a/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py b/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py new file mode 100644 index 0000000000..76243fe227 --- /dev/null +++ b/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py @@ -0,0 +1,301 @@ +""" +Canonical post-fix expectations for the reasoning_effort grid v4 sweep. + +The QA sweep on https://github.com/BerriAI/litellm/pull/27039#issuecomment-4363363610 +covered 21 (provider x model) combos x 11 effort values (231 cells). The follow-up +PR https://github.com/BerriAI/litellm/pull/27074 closed nine bugs surfaced by that +sweep. This module encodes the post-fix expectations as a small rule set keyed by +(model_mode, effort) and per-model capability overrides, then expands them across +the model x effort matrix per route. +""" + +from dataclasses import dataclass, field +from typing import Dict, FrozenSet, List, Optional, Tuple + + +OMIT = object() + + +@dataclass(frozen=True) +class CellExpectation: + """Expected post-fix behavior for a single grid cell.""" + + status: int + thinking_type: object + output_config_effort: object = OMIT + thinking_budget_tokens: object = OMIT + max_tokens: object = OMIT + + +@dataclass(frozen=True) +class ModelEntry: + alias: str + model: str + mode: str + extra_params: Tuple[Tuple[str, str], ...] = field(default_factory=tuple) + required_env: FrozenSet[str] = field(default_factory=frozenset) + caps: FrozenSet[str] = field(default_factory=frozenset) + + def params(self) -> Dict[str, str]: + return dict(self.extra_params) + + +EFFORTS: Tuple[str, ...] = ( + "__omit__", + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + "disabled", + "invalid", + "", +) + +_BUDGET_TOKENS: Dict[str, int] = { + "minimal": 1024, + "low": 1024, + "medium": 2048, + "high": 4096, +} + +_ADAPTIVE_EFFORT_LABEL: Dict[str, str] = { + "minimal": "low", + "low": "low", + "medium": "medium", + "high": "high", + "xhigh": "xhigh", + "max": "max", +} + +_BAD_REQUEST_EFFORTS: FrozenSet[str] = frozenset({"disabled", "invalid", ""}) + + +def expected(model: ModelEntry, effort: str) -> CellExpectation: + """Compute the post-fix expected cell for a (model, effort) pair.""" + if effort in ("__omit__", "none"): + if model.mode == "budget": + return CellExpectation(status=200, thinking_type=OMIT, max_tokens=8192) + return CellExpectation(status=200, thinking_type=OMIT) + + if effort in _BAD_REQUEST_EFFORTS: + return CellExpectation(status=400, thinking_type=OMIT) + + if effort in ("xhigh", "max"): + cap = f"supports_{effort}_reasoning_effort" + if cap not in model.caps: + return CellExpectation(status=400, thinking_type=OMIT) + + if model.mode == "adaptive": + return CellExpectation( + status=200, + thinking_type="adaptive", + output_config_effort=_ADAPTIVE_EFFORT_LABEL[effort], + ) + + return CellExpectation( + status=200, + thinking_type="enabled", + thinking_budget_tokens=_BUDGET_TOKENS[effort], + max_tokens=8192, + ) + + +_ANTHROPIC_REQ = frozenset({"ANTHROPIC_API_KEY"}) +_AZURE_FOUNDRY_REQ = frozenset({"AZURE_FOUNDRY_API_BASE", "AZURE_FOUNDRY_API_KEY"}) +_VERTEX_REQ = frozenset({"VERTEX_PROJECT"}) +_BEDROCK_REQ = frozenset({"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"}) + + +_CAPS_OPUS_4_7: FrozenSet[str] = frozenset( + {"supports_xhigh_reasoning_effort", "supports_max_reasoning_effort"} +) +_CAPS_OPUS_4_6: FrozenSet[str] = frozenset({"supports_max_reasoning_effort"}) +_CAPS_NONE: FrozenSet[str] = frozenset() + + +ANTHROPIC_DIRECT_MODELS: Tuple[ModelEntry, ...] = ( + ModelEntry( + alias="claude-opus-4-7", + model="anthropic/claude-opus-4-7", + mode="adaptive", + required_env=_ANTHROPIC_REQ, + caps=_CAPS_OPUS_4_7, + ), + ModelEntry( + alias="claude-sonnet-4-6", + model="anthropic/claude-sonnet-4-6", + mode="adaptive", + required_env=_ANTHROPIC_REQ, + caps=_CAPS_NONE, + ), + ModelEntry( + alias="claude-haiku-4-5", + model="anthropic/claude-haiku-4-5", + mode="budget", + required_env=_ANTHROPIC_REQ, + caps=_CAPS_NONE, + ), +) + + +AZURE_AI_MODELS: Tuple[ModelEntry, ...] = ( + ModelEntry( + alias="azure-claude-opus-4-7", + model="azure_ai/claude-opus-4-7", + mode="adaptive", + required_env=_AZURE_FOUNDRY_REQ, + caps=_CAPS_OPUS_4_7, + ), + ModelEntry( + alias="azure-claude-opus-4-6", + model="azure_ai/claude-opus-4-6", + mode="adaptive", + required_env=_AZURE_FOUNDRY_REQ, + caps=_CAPS_OPUS_4_6, + ), + ModelEntry( + alias="azure-claude-sonnet-4-6", + model="azure_ai/claude-sonnet-4-6", + mode="adaptive", + required_env=_AZURE_FOUNDRY_REQ, + caps=_CAPS_NONE, + ), + ModelEntry( + alias="azure-claude-haiku-4-5", + model="azure_ai/claude-haiku-4-5", + mode="budget", + required_env=_AZURE_FOUNDRY_REQ, + caps=_CAPS_NONE, + ), +) + + +VERTEX_AI_MODELS: Tuple[ModelEntry, ...] = ( + ModelEntry( + alias="vertex-claude-opus-4-7", + model="vertex_ai/claude-opus-4-7", + mode="adaptive", + extra_params=(("vertex_location", "global"),), + required_env=_VERTEX_REQ, + caps=_CAPS_OPUS_4_7, + ), + ModelEntry( + alias="vertex-claude-opus-4-6", + model="vertex_ai/claude-opus-4-6", + mode="adaptive", + extra_params=(("vertex_location", "us-east5"),), + required_env=_VERTEX_REQ, + caps=_CAPS_OPUS_4_6, + ), + ModelEntry( + alias="vertex-claude-sonnet-4-6", + model="vertex_ai/claude-sonnet-4-6", + mode="adaptive", + extra_params=(("vertex_location", "us-east5"),), + required_env=_VERTEX_REQ, + caps=_CAPS_NONE, + ), + ModelEntry( + alias="vertex-claude-haiku-4-5", + model="vertex_ai/claude-haiku-4-5", + mode="budget", + extra_params=(("vertex_location", "us-east5"),), + required_env=_VERTEX_REQ, + caps=_CAPS_NONE, + ), +) + + +BEDROCK_CONVERSE_MODELS: Tuple[ModelEntry, ...] = ( + ModelEntry( + alias="bedrock-claude-opus-4-7", + model="bedrock/converse/us.anthropic.claude-opus-4-7", + mode="adaptive", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_OPUS_4_7, + ), + ModelEntry( + alias="bedrock-claude-opus-4-6", + model="bedrock/converse/us.anthropic.claude-opus-4-6-v1", + mode="adaptive", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_OPUS_4_6, + ), + ModelEntry( + alias="bedrock-claude-sonnet-4-6", + model="bedrock/converse/us.anthropic.claude-sonnet-4-6", + mode="adaptive", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_NONE, + ), + ModelEntry( + alias="bedrock-claude-sonnet-4-5", + model="bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + mode="budget", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_NONE, + ), +) + + +BEDROCK_INVOKE_CHAT_MODELS: Tuple[ModelEntry, ...] = ( + ModelEntry( + alias="bedrock-invoke-claude-opus-4-6", + model="bedrock/invoke/us.anthropic.claude-opus-4-6-v1", + mode="adaptive", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_OPUS_4_6, + ), + ModelEntry( + alias="bedrock-invoke-claude-sonnet-4-6", + model="bedrock/invoke/us.anthropic.claude-sonnet-4-6", + mode="adaptive", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_NONE, + ), + ModelEntry( + alias="bedrock-invoke-claude-opus-4-5", + model="bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0", + mode="budget", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_NONE, + ), +) + + +BEDROCK_INVOKE_MESSAGES_MODELS: Tuple[ModelEntry, ...] = BEDROCK_INVOKE_CHAT_MODELS + + +@dataclass(frozen=True) +class Route: + name: str + models: Tuple[ModelEntry, ...] + + +ROUTES: Tuple[Route, ...] = ( + Route("anthropic_direct", ANTHROPIC_DIRECT_MODELS), + Route("azure_ai", AZURE_AI_MODELS), + Route("vertex_ai", VERTEX_AI_MODELS), + Route("bedrock_converse", BEDROCK_CONVERSE_MODELS), + Route("bedrock_invoke_chat", BEDROCK_INVOKE_CHAT_MODELS), + Route("bedrock_invoke_messages", BEDROCK_INVOKE_MESSAGES_MODELS), +) + + +def all_cells() -> List[Tuple[str, ModelEntry, str, CellExpectation]]: + cells: List[Tuple[str, ModelEntry, str, CellExpectation]] = [] + for route in ROUTES: + for model in route.models: + for effort in EFFORTS: + cells.append((route.name, model, effort, expected(model, effort))) + return cells diff --git a/tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py b/tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py new file mode 100644 index 0000000000..9eec572988 --- /dev/null +++ b/tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py @@ -0,0 +1,230 @@ +""" +End-to-end grid v4 regression suite for reasoning_effort mapping across +Anthropic-backed routes. + +Encodes the 21 (provider x model) x 11 effort matrix (231 cells) from the +QA sweep on https://github.com/BerriAI/litellm/pull/27039#issuecomment-4363363610 +that the fix in https://github.com/BerriAI/litellm/pull/27074 was validated +against. Each cell asserts: + + - Wire body shape captured pre-call (thinking.type, output_config.effort, + thinking.budget_tokens, max_tokens) -- the regression signal for silent + drops/strips anywhere in the transformation chain. + - Status code returned by LiteLLM (200 vs BadRequestError -> 400) -- the + regression signal for clean-error vs leaked-500 mappings. + +Hits real provider endpoints. Each route is skipped at runtime when its +required env vars are absent, so PR builds without provider credentials no-op +gracefully. +""" + +import os +from typing import Any, Dict, List, Optional, Tuple + +import pytest + +import litellm +from litellm.exceptions import BadRequestError + +from .grid_spec import ( + OMIT, + ROUTES, + CellExpectation, + ModelEntry, + all_cells, +) + + +_PROMPT_MESSAGES: List[Dict[str, str]] = [ + {"role": "user", "content": "Step by step, calculate 47 * 53. Show your work."} +] + + +def _required_env_missing(model: ModelEntry) -> Optional[str]: + missing = [key for key in model.required_env if not os.environ.get(key)] + if missing: + return "missing env: " + ", ".join(sorted(missing)) + return None + + +def _max_tokens_for(model: ModelEntry) -> int: + return 200 if model.mode == "adaptive" else 8192 + + +def _build_completion_kwargs(model: ModelEntry, effort: str) -> Dict[str, Any]: + kwargs: Dict[str, Any] = { + "model": model.model, + "messages": _PROMPT_MESSAGES, + "max_tokens": _max_tokens_for(model), + } + kwargs.update(model.params()) + if effort != "__omit__": + kwargs["reasoning_effort"] = effort + if model.model.startswith("vertex_ai/"): + kwargs["vertex_project"] = os.environ.get( + "VERTEX_PROJECT", "vertex-check-481318" + ) + if model.model.startswith("azure_ai/"): + kwargs["api_base"] = os.environ["AZURE_FOUNDRY_API_BASE"] + kwargs["api_key"] = os.environ["AZURE_FOUNDRY_API_KEY"] + return kwargs + + +def _build_messages_kwargs(model: ModelEntry, effort: str) -> Dict[str, Any]: + kwargs = _build_completion_kwargs(model, effort) + return kwargs + + +def _converse_subbody(body: Dict[str, Any]) -> Dict[str, Any]: + """Return the dict that holds thinking/output_config for a Converse wire body.""" + return body.get("additionalModelRequestFields", body) + + +def _max_tokens_from_body(body: Dict[str, Any], route_name: str) -> Optional[int]: + if route_name == "bedrock_converse": + return body.get("inferenceConfig", {}).get("maxTokens") + return body.get("max_tokens") + + +def _assert_cell( + route_name: str, + body: Optional[Dict[str, Any]], + status: int, + cell: CellExpectation, +) -> None: + assert status == cell.status, f"expected status={cell.status}, got status={status}" + + if cell.status != 200: + # Bad-request paths short-circuit before the wire body matters. + return + + assert body is not None, "wire body was not captured for a 200-status cell" + subbody = _converse_subbody(body) if route_name == "bedrock_converse" else body + thinking = subbody.get("thinking") + output_config = subbody.get("output_config") + + if cell.thinking_type is OMIT: + assert thinking is None, f"expected thinking omitted, got {thinking!r}" + else: + assert thinking is not None, "expected thinking present, got omit" + assert thinking.get("type") == cell.thinking_type, ( + f"expected thinking.type={cell.thinking_type!r}, " + f"got {thinking.get('type')!r}" + ) + + if cell.output_config_effort is OMIT: + assert ( + output_config is None or "effort" not in output_config + ), f"expected output_config.effort omitted, got {output_config!r}" + else: + assert output_config is not None, ( + f"expected output_config.effort={cell.output_config_effort!r}, " + "got output_config omitted" + ) + assert output_config.get("effort") == cell.output_config_effort, ( + f"expected output_config.effort={cell.output_config_effort!r}, " + f"got {output_config.get('effort')!r}" + ) + + if cell.thinking_budget_tokens is not OMIT: + assert thinking is not None + assert thinking.get("budget_tokens") == cell.thinking_budget_tokens, ( + f"expected thinking.budget_tokens={cell.thinking_budget_tokens!r}, " + f"got {thinking.get('budget_tokens')!r}" + ) + + if cell.max_tokens is not OMIT: + wire_max = _max_tokens_from_body(body, route_name) + assert ( + wire_max == cell.max_tokens + ), f"expected max_tokens={cell.max_tokens!r}, got {wire_max!r}" + + +_PARAMS: List[Tuple[str, ModelEntry, str, CellExpectation]] = all_cells() + + +def _cell_id(case: Tuple[str, ModelEntry, str, CellExpectation]) -> str: + route_name, model, effort, _ = case + effort_label = "__empty__" if effort == "" else effort + return f"{route_name}-{model.alias}-{effort_label}" + + +_PARAM_IDS: List[str] = [_cell_id(case) for case in _PARAMS] + + +async def _call_chat(model: ModelEntry, effort: str) -> Tuple[int, Optional[Exception]]: + kwargs = _build_completion_kwargs(model, effort) + try: + await litellm.acompletion(**kwargs) + return 200, None + except BadRequestError as exc: + return 400, exc + except Exception as exc: + return 500, exc + + +async def _call_messages( + model: ModelEntry, effort: str +) -> Tuple[int, Optional[Exception]]: + kwargs = _build_messages_kwargs(model, effort) + try: + await litellm.messages.acreate(**kwargs) + return 200, None + except BadRequestError as exc: + return 400, exc + except Exception as exc: + return 500, exc + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route_name", "model", "effort", "cell"), _PARAMS, ids=_PARAM_IDS +) +async def test_reasoning_effort_grid_v4( + route_name: str, + model: ModelEntry, + effort: str, + cell: CellExpectation, + wire_capture, +) -> None: + skip_reason = _required_env_missing(model) + if skip_reason: + pytest.skip(skip_reason) + + if route_name == "bedrock_invoke_messages": + status, exc = await _call_messages(model, effort) + else: + status, exc = await _call_chat(model, effort) + + record = wire_capture.latest() + body = record["body"] if record else None + + try: + _assert_cell(route_name, body, status, cell) + except AssertionError: + if exc is not None: + raise AssertionError( + f"underlying exception ({type(exc).__name__}): {exc}" + ) from None + raise + + +def test_grid_v4_cell_count() -> None: + """Guard against accidental drops or duplicates in the grid spec.""" + assert len(_PARAMS) == 21 * 11, ( + f"expected 231 cells (21 provider x model combos x 11 efforts), " + f"got {len(_PARAMS)}" + ) + + +def test_grid_v4_route_coverage() -> None: + """The grid must cover every route the original QA sweep covered.""" + route_names = {route.name for route in ROUTES} + assert route_names == { + "anthropic_direct", + "azure_ai", + "vertex_ai", + "bedrock_converse", + "bedrock_invoke_chat", + "bedrock_invoke_messages", + } From 432742778a0b43eed9cc72f7619e4b331e7c1c1d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 16 May 2026 02:58:20 +0000 Subject: [PATCH 02/10] fix(reasoning_effort_grid_v4): cleanup unused fixture, parse converse body, guard budget tokens - Remove unused vertex_credentials_path fixture (and now-unused os import) from conftest.py. - Parse Bedrock Converse complete_input_dict (logged as a JSON string by converse_handler.py) before passing to _assert_cell, so dict accessors work uniformly across routes. - Extend _BUDGET_TOKENS with xhigh and max entries so the budget-mode branch in expected() cannot KeyError if a future budget model gains the matching cap. Co-authored-by: Yassin Kortam --- .../test_litellm/reasoning_effort_grid_v4/conftest.py | 10 ---------- .../test_litellm/reasoning_effort_grid_v4/grid_spec.py | 2 ++ .../reasoning_effort_grid_v4/test_grid_v4.py | 6 ++++++ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/test_litellm/reasoning_effort_grid_v4/conftest.py b/tests/test_litellm/reasoning_effort_grid_v4/conftest.py index 5e1a4e0e97..e5f0377856 100644 --- a/tests/test_litellm/reasoning_effort_grid_v4/conftest.py +++ b/tests/test_litellm/reasoning_effort_grid_v4/conftest.py @@ -1,6 +1,5 @@ """Shared fixtures for the reasoning_effort grid v4 e2e suite.""" -import os from typing import Any, Dict, List, Optional import pytest @@ -50,12 +49,3 @@ def wire_capture(): yield capture finally: litellm.callbacks = previous - - -@pytest.fixture(scope="session") -def vertex_credentials_path() -> Optional[str]: - """Resolve a usable Vertex credentials file path or None.""" - path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") - if path and os.path.exists(path): - return path - return None diff --git a/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py b/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py index 76243fe227..7ba676118a 100644 --- a/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py +++ b/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py @@ -59,6 +59,8 @@ _BUDGET_TOKENS: Dict[str, int] = { "low": 1024, "medium": 2048, "high": 4096, + "xhigh": 8192, + "max": 16384, } _ADAPTIVE_EFFORT_LABEL: Dict[str, str] = { diff --git a/tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py b/tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py index 9eec572988..7e74cf1f2f 100644 --- a/tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py +++ b/tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py @@ -18,6 +18,7 @@ required env vars are absent, so PR builds without provider credentials no-op gracefully. """ +import json import os from typing import Any, Dict, List, Optional, Tuple @@ -198,6 +199,11 @@ async def test_reasoning_effort_grid_v4( record = wire_capture.latest() body = record["body"] if record else None + # Bedrock Converse logs `complete_input_dict` as a JSON string (see + # litellm/llms/bedrock/chat/converse_handler.py); parse it so the dict + # accessors in `_assert_cell` work uniformly across routes. + if route_name == "bedrock_converse" and isinstance(body, str): + body = json.loads(body) try: _assert_cell(route_name, body, status, cell) From 51dff1ff79d4f524b4ba35c3836af16be855289d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 16 May 2026 03:12:20 +0000 Subject: [PATCH 03/10] fix(reasoning_effort_grid_v4): grant sonnet-4-6 entries the max-effort cap The runtime _validate_effort_for_model allows effort='max' for any Claude 4.6 model (opus or sonnet), and model_prices_and_context_window sets supports_max_reasoning_effort: true for claude-sonnet-4-6. The grid spec previously gave sonnet-4-6 entries _CAPS_NONE, so expected() returned status=400 for effort='max', which mismatched the runtime's status=200 and caused 6 cells (one per route) to fail. Rename _CAPS_OPUS_4_6 to _CAPS_4_6 (since the cap set is shared by opus and sonnet 4.6) and assign it to all sonnet-4-6 entries. Co-authored-by: Yassin Kortam --- .../reasoning_effort_grid_v4/grid_spec.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py b/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py index 7ba676118a..cbd2cb5a9e 100644 --- a/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py +++ b/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py @@ -114,7 +114,7 @@ _BEDROCK_REQ = frozenset({"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"}) _CAPS_OPUS_4_7: FrozenSet[str] = frozenset( {"supports_xhigh_reasoning_effort", "supports_max_reasoning_effort"} ) -_CAPS_OPUS_4_6: FrozenSet[str] = frozenset({"supports_max_reasoning_effort"}) +_CAPS_4_6: FrozenSet[str] = frozenset({"supports_max_reasoning_effort"}) _CAPS_NONE: FrozenSet[str] = frozenset() @@ -131,7 +131,7 @@ ANTHROPIC_DIRECT_MODELS: Tuple[ModelEntry, ...] = ( model="anthropic/claude-sonnet-4-6", mode="adaptive", required_env=_ANTHROPIC_REQ, - caps=_CAPS_NONE, + caps=_CAPS_4_6, ), ModelEntry( alias="claude-haiku-4-5", @@ -156,14 +156,14 @@ AZURE_AI_MODELS: Tuple[ModelEntry, ...] = ( model="azure_ai/claude-opus-4-6", mode="adaptive", required_env=_AZURE_FOUNDRY_REQ, - caps=_CAPS_OPUS_4_6, + caps=_CAPS_4_6, ), ModelEntry( alias="azure-claude-sonnet-4-6", model="azure_ai/claude-sonnet-4-6", mode="adaptive", required_env=_AZURE_FOUNDRY_REQ, - caps=_CAPS_NONE, + caps=_CAPS_4_6, ), ModelEntry( alias="azure-claude-haiku-4-5", @@ -190,7 +190,7 @@ VERTEX_AI_MODELS: Tuple[ModelEntry, ...] = ( mode="adaptive", extra_params=(("vertex_location", "us-east5"),), required_env=_VERTEX_REQ, - caps=_CAPS_OPUS_4_6, + caps=_CAPS_4_6, ), ModelEntry( alias="vertex-claude-sonnet-4-6", @@ -198,7 +198,7 @@ VERTEX_AI_MODELS: Tuple[ModelEntry, ...] = ( mode="adaptive", extra_params=(("vertex_location", "us-east5"),), required_env=_VERTEX_REQ, - caps=_CAPS_NONE, + caps=_CAPS_4_6, ), ModelEntry( alias="vertex-claude-haiku-4-5", @@ -226,7 +226,7 @@ BEDROCK_CONVERSE_MODELS: Tuple[ModelEntry, ...] = ( mode="adaptive", extra_params=(("aws_region_name", "us-east-1"),), required_env=_BEDROCK_REQ, - caps=_CAPS_OPUS_4_6, + caps=_CAPS_4_6, ), ModelEntry( alias="bedrock-claude-sonnet-4-6", @@ -234,7 +234,7 @@ BEDROCK_CONVERSE_MODELS: Tuple[ModelEntry, ...] = ( mode="adaptive", extra_params=(("aws_region_name", "us-east-1"),), required_env=_BEDROCK_REQ, - caps=_CAPS_NONE, + caps=_CAPS_4_6, ), ModelEntry( alias="bedrock-claude-sonnet-4-5", @@ -254,7 +254,7 @@ BEDROCK_INVOKE_CHAT_MODELS: Tuple[ModelEntry, ...] = ( mode="adaptive", extra_params=(("aws_region_name", "us-east-1"),), required_env=_BEDROCK_REQ, - caps=_CAPS_OPUS_4_6, + caps=_CAPS_4_6, ), ModelEntry( alias="bedrock-invoke-claude-sonnet-4-6", @@ -262,7 +262,7 @@ BEDROCK_INVOKE_CHAT_MODELS: Tuple[ModelEntry, ...] = ( mode="adaptive", extra_params=(("aws_region_name", "us-east-1"),), required_env=_BEDROCK_REQ, - caps=_CAPS_NONE, + caps=_CAPS_4_6, ), ModelEntry( alias="bedrock-invoke-claude-opus-4-5", From f77324d766958286cbaa1c6313f36c0ca98461e3 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 16 May 2026 03:21:28 +0000 Subject: [PATCH 04/10] refactor(tests): move reasoning_effort grid suite under llm_translation, drop v4 naming - Drop the "v4" suffix throughout: it referred to the QA sweep iteration, not this test suite. There's only one regression suite, so just call it reasoning_effort_grid. - Move tests/test_litellm/reasoning_effort_grid_v4/ -> tests/llm_translation/ reasoning_effort_grid/. Two reasons: 1. The parent tests/test_litellm/conftest.py installs an autouse fixture (isolate_host_aws_config) that clears every AWS_* env var before each test, which would silently skip every Bedrock cell. 2. tests/llm_translation/conftest.py already wires up the Redis-backed VCR persister and auto-applies @pytest.mark.vcr to every collected item via apply_vcr_auto_marker_to_items. Living under that conftest means the suite gets cassette replay for free -- first CI run with provider creds records 231 cassettes, every subsequent run replays them with no live spend. - Trim the suite's own conftest down to just the wire_capture fixture; the inherited llm_translation conftest covers the VCR plumbing. - Drop the dedicated reasoning_effort_grid_v4_e2e CircleCI job. The existing llm_translation_testing job globs tests/llm_translation/**/test_*.py, so the suite is gated by an existing job with no new wiring. --- .circleci/config.yml | 44 ------------------- .../reasoning_effort_grid}/__init__.py | 0 .../reasoning_effort_grid}/conftest.py | 30 +++++-------- .../reasoning_effort_grid}/grid_spec.py | 15 ++++--- .../test_reasoning_effort_grid.py} | 19 ++++---- 5 files changed, 31 insertions(+), 77 deletions(-) rename tests/{test_litellm/reasoning_effort_grid_v4 => llm_translation/reasoning_effort_grid}/__init__.py (100%) rename tests/{test_litellm/reasoning_effort_grid_v4 => llm_translation/reasoning_effort_grid}/conftest.py (61%) rename tests/{test_litellm/reasoning_effort_grid_v4 => llm_translation/reasoning_effort_grid}/grid_spec.py (94%) rename tests/{test_litellm/reasoning_effort_grid_v4/test_grid_v4.py => llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py} (91%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3dd1221c12..9a5b6da77f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -576,48 +576,6 @@ jobs: no_output_timeout: 15m # Store test results - - store_test_results: - path: test-results - reasoning_effort_grid_v4_e2e: - docker: - - *python312_image - working_directory: ~/project - resource_class: large - - steps: - - checkout - - setup_google_dns - - install_uv - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - # Grid v4 exercises reasoning_effort mapping against real Anthropic, - # Azure AI Foundry, Vertex AI, Bedrock Converse, and Bedrock Invoke - # endpoints. Per-route cells pytest-skip themselves when the matching - # provider env vars are absent, so PRs without credentials no-op. - - run: - name: Run reasoning_effort grid v4 e2e suite - command: | - mkdir -p test-results - uv run --no-sync python -m pytest \ - tests/test_litellm/reasoning_effort_grid_v4/ \ - -v \ - --junitxml=test-results/junit.xml \ - --durations=20 \ - -n 4 \ - --timeout=180 --timeout_method=thread \ - --retries 2 --retry-delay 5 \ - --max-worker-restart=5 - no_output_timeout: 20m - - store_test_results: path: test-results realtime_translation_testing: @@ -2661,8 +2619,6 @@ workflows: filters: *main_branches - llm_translation_testing: filters: *main_branches - - reasoning_effort_grid_v4_e2e: - filters: *main_branches - realtime_translation_testing: filters: *main_branches - agent_testing: diff --git a/tests/test_litellm/reasoning_effort_grid_v4/__init__.py b/tests/llm_translation/reasoning_effort_grid/__init__.py similarity index 100% rename from tests/test_litellm/reasoning_effort_grid_v4/__init__.py rename to tests/llm_translation/reasoning_effort_grid/__init__.py diff --git a/tests/test_litellm/reasoning_effort_grid_v4/conftest.py b/tests/llm_translation/reasoning_effort_grid/conftest.py similarity index 61% rename from tests/test_litellm/reasoning_effort_grid_v4/conftest.py rename to tests/llm_translation/reasoning_effort_grid/conftest.py index 5e1a4e0e97..aad85307cc 100644 --- a/tests/test_litellm/reasoning_effort_grid_v4/conftest.py +++ b/tests/llm_translation/reasoning_effort_grid/conftest.py @@ -1,6 +1,12 @@ -"""Shared fixtures for the reasoning_effort grid v4 e2e suite.""" +"""Shared fixtures for the reasoning_effort grid e2e suite. + +VCR wiring (Redis-backed cassette persister, auto-application of +``@pytest.mark.vcr`` to every collected item, cassette-cache health summary) +is inherited from ``tests/llm_translation/conftest.py``. This file only +contributes the ``wire_capture`` fixture, which records the wire body +LiteLLM sends upstream so each cell can inspect it. +""" -import os from typing import Any, Dict, List, Optional import pytest @@ -12,10 +18,10 @@ from litellm.integrations.custom_logger import CustomLogger class _WireBodyCapture(CustomLogger): """Pre-call hook that records the outgoing wire body LiteLLM sends upstream. - `complete_input_dict` is the fully transformed provider request as set by - every provider transformation in `litellm/llms/**`. Capturing it here means - a regression anywhere in the transformation chain (strip, rename, drop) - surfaces as an assertion failure on the cell that depends on it. + ``complete_input_dict`` is the fully transformed provider request as set + by every provider transformation in ``litellm/llms/**``. Capturing it here + means a regression anywhere in the transformation chain (strip, rename, + drop) surfaces as an assertion failure on the cell that depends on it. """ def __init__(self) -> None: @@ -37,9 +43,6 @@ class _WireBodyCapture(CustomLogger): def latest(self) -> Optional[Dict[str, Any]]: return self.records[-1] if self.records else None - def reset(self) -> None: - self.records.clear() - @pytest.fixture() def wire_capture(): @@ -50,12 +53,3 @@ def wire_capture(): yield capture finally: litellm.callbacks = previous - - -@pytest.fixture(scope="session") -def vertex_credentials_path() -> Optional[str]: - """Resolve a usable Vertex credentials file path or None.""" - path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") - if path and os.path.exists(path): - return path - return None diff --git a/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py b/tests/llm_translation/reasoning_effort_grid/grid_spec.py similarity index 94% rename from tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py rename to tests/llm_translation/reasoning_effort_grid/grid_spec.py index 76243fe227..5185dfc2da 100644 --- a/tests/test_litellm/reasoning_effort_grid_v4/grid_spec.py +++ b/tests/llm_translation/reasoning_effort_grid/grid_spec.py @@ -1,12 +1,13 @@ """ -Canonical post-fix expectations for the reasoning_effort grid v4 sweep. +Canonical post-fix expectations for the reasoning_effort grid sweep. -The QA sweep on https://github.com/BerriAI/litellm/pull/27039#issuecomment-4363363610 -covered 21 (provider x model) combos x 11 effort values (231 cells). The follow-up -PR https://github.com/BerriAI/litellm/pull/27074 closed nine bugs surfaced by that -sweep. This module encodes the post-fix expectations as a small rule set keyed by -(model_mode, effort) and per-model capability overrides, then expands them across -the model x effort matrix per route. +The original QA sweep on +https://github.com/BerriAI/litellm/pull/27039#issuecomment-4363363610 +covered 21 (provider x model) combos x 11 effort values (231 cells). The +follow-up PR https://github.com/BerriAI/litellm/pull/27074 closed nine bugs +surfaced by that sweep. This module encodes the post-fix expectations as a +small rule set keyed by (model_mode, effort) and per-model capability +overrides, then expands them across the model x effort matrix per route. """ from dataclasses import dataclass, field diff --git a/tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py similarity index 91% rename from tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py rename to tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py index 9eec572988..6925a2aea9 100644 --- a/tests/test_litellm/reasoning_effort_grid_v4/test_grid_v4.py +++ b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py @@ -1,6 +1,6 @@ """ -End-to-end grid v4 regression suite for reasoning_effort mapping across -Anthropic-backed routes. +End-to-end regression suite for reasoning_effort mapping across the +Anthropic-backed routes covered by the original QA sweep. Encodes the 21 (provider x model) x 11 effort matrix (231 cells) from the QA sweep on https://github.com/BerriAI/litellm/pull/27039#issuecomment-4363363610 @@ -13,9 +13,12 @@ against. Each cell asserts: - Status code returned by LiteLLM (200 vs BadRequestError -> 400) -- the regression signal for clean-error vs leaked-500 mappings. -Hits real provider endpoints. Each route is skipped at runtime when its -required env vars are absent, so PR builds without provider credentials no-op -gracefully. +Calls go to real provider endpoints, but the parent +``tests/llm_translation/conftest.py`` auto-applies ``@pytest.mark.vcr`` to +every collected item, so first run records cassettes (Redis-backed) and +subsequent CI runs replay them with no live spend. Each route still skips at +runtime when its required env vars are absent, so PR builds without provider +credentials no-op gracefully. """ import os @@ -180,7 +183,7 @@ async def _call_messages( @pytest.mark.parametrize( ("route_name", "model", "effort", "cell"), _PARAMS, ids=_PARAM_IDS ) -async def test_reasoning_effort_grid_v4( +async def test_reasoning_effort_grid( route_name: str, model: ModelEntry, effort: str, @@ -209,7 +212,7 @@ async def test_reasoning_effort_grid_v4( raise -def test_grid_v4_cell_count() -> None: +def test_grid_cell_count() -> None: """Guard against accidental drops or duplicates in the grid spec.""" assert len(_PARAMS) == 21 * 11, ( f"expected 231 cells (21 provider x model combos x 11 efforts), " @@ -217,7 +220,7 @@ def test_grid_v4_cell_count() -> None: ) -def test_grid_v4_route_coverage() -> None: +def test_grid_route_coverage() -> None: """The grid must cover every route the original QA sweep covered.""" route_names = {route.name for route in ROUTES} assert route_names == { From 90cdbb92d7787ea962f7e1b216ab41ea7d1f499f Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 16 May 2026 06:59:08 +0000 Subject: [PATCH 05/10] fix(tests): use litellm.anthropic_messages entrypoint + drop unstable openapi field Two CI failures, both pre-existing in different ways: 1. reasoning_effort_grid: all 33 bedrock_invoke_messages cells failed with AttributeError("module 'litellm' has no attribute 'messages'"). litellm exposes the async Anthropic Messages entrypoint as litellm.anthropic_messages (via "from .llms.anthropic.experimental_pass_through.messages.handler import *" in litellm/__init__.py), not litellm.messages.acreate. Swap the call. 2. tests/test_litellm/interactions/test_openapi_compliance.py::TestResponseCompliance::test_interaction_response_fields asserts the live Google spec contains "steps". Google's spec has churned through "outputs" -> "steps" -> neither, and presently carries neither. The test broke on main as soon as upstream dropped "steps"; pulling the key off the assert list realigns the test with the live schema. Re-add the per-turn output field once upstream stabilizes on a name. The openapi-compliance fix doesn't belong to this PR conceptually but is included here per request to unblock CI before the morning. --- .../reasoning_effort_grid/test_reasoning_effort_grid.py | 2 +- .../test_litellm/interactions/test_openapi_compliance.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py index e15d449287..43f0fa1dbe 100644 --- a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py +++ b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py @@ -172,7 +172,7 @@ async def _call_messages( ) -> Tuple[int, Optional[Exception]]: kwargs = _build_messages_kwargs(model, effort) try: - await litellm.messages.acreate(**kwargs) + await litellm.anthropic_messages(**kwargs) return 200, None except BadRequestError as exc: return 400, exc diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index 11d61d4c82..ce74bf150a 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -157,15 +157,17 @@ class TestResponseCompliance: # Check CreateModelInteractionParams which includes output fields schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"] - # Output fields (readOnly). Google renamed `outputs` → `steps` in the - # upstream spec; keep this list aligned with the live schema. + # Output fields (readOnly). Google's live spec has churned through + # both `outputs` and `steps` for the per-turn output array and at the + # moment carries neither -- only the stable response-level fields + # below are guaranteed. Re-add the per-turn key once upstream + # stabilizes on a name. output_fields = [ "id", "status", "created", "updated", "role", - "steps", "usage", ] From e29ea53c31fe41fb16e69f640d2d05bb16cc2dc6 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 16 May 2026 07:32:25 +0000 Subject: [PATCH 06/10] fix(reasoning_effort_grid): classify status by exception status_code, not class The anthropic_messages route wraps client-side BadRequestError as AnthropicError (a BaseLLMException subclass) with status_code=400, so "except BadRequestError" missed those cells and they fell through to the generic Exception arm, returning 500 instead of the expected 400. Replace the isinstance-on-BadRequestError check with a tiny classifier that prefers BadRequestError membership, then falls back to the exception's status_code attribute (set by every BaseLLMException subclass), then 500. Apply to both _call_chat and _call_messages for consistency. Fixes the 13 CircleCI llm_translation_testing failures on bedrock_invoke_messages cells where the effort was disabled / invalid / empty / xhigh-on-unsupported / max-on-unsupported. --- .../test_reasoning_effort_grid.py | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py index 43f0fa1dbe..a815eb861e 100644 --- a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py +++ b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py @@ -156,15 +156,30 @@ def _cell_id(case: Tuple[str, ModelEntry, str, CellExpectation]) -> str: _PARAM_IDS: List[str] = [_cell_id(case) for case in _PARAMS] +def _classify_status(exc: Exception) -> int: + """Map an exception to the HTTP status the QA grid would have observed. + + The Anthropic Messages route (litellm.anthropic_messages) wraps client-side + BadRequestError as ``AnthropicError`` with ``status_code=400`` rather than + re-raising the BadRequestError class directly, so isinstance() alone misses + those cells. Read the ``status_code`` attribute when present (set by every + BaseLLMException subclass) and fall through to 500 otherwise. + """ + if isinstance(exc, BadRequestError): + return 400 + code = getattr(exc, "status_code", None) + if isinstance(code, int): + return code + return 500 + + async def _call_chat(model: ModelEntry, effort: str) -> Tuple[int, Optional[Exception]]: kwargs = _build_completion_kwargs(model, effort) try: await litellm.acompletion(**kwargs) return 200, None - except BadRequestError as exc: - return 400, exc except Exception as exc: - return 500, exc + return _classify_status(exc), exc async def _call_messages( @@ -174,10 +189,8 @@ async def _call_messages( try: await litellm.anthropic_messages(**kwargs) return 200, None - except BadRequestError as exc: - return 400, exc except Exception as exc: - return 500, exc + return _classify_status(exc), exc @pytest.mark.asyncio From 2b00ea9ee486ee86300a41a2d85df214ab1e86d1 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 16 May 2026 07:47:25 +0000 Subject: [PATCH 07/10] test(ci): skip Fireworks tests on 404 + Gemini image-size test on 429 Four pre-existing flakes on main that gate this branch's workflow even though they're unrelated to the reasoning_effort_grid suite: 1. tests/local_testing/test_completion.py::test_completion_fireworks_ai 2. tests/local_testing/test_completion_cost.py::test_completion_cost_fireworks_ai[fireworks_ai/llama-v3p3-70b-instruct] 3. tests/llm_translation/test_fireworks_ai_translation.py::test_document_inlining_example[False] The Fireworks-hosted `llama-v3p3-70b-instruct` deployment is currently returning 404 "Model not found, inaccessible, and/or not deployed". These tests pass when the model is deployed; the issue is upstream capacity, not our code path. Wrap the live call in a try/except that pytest.skip's on litellm.NotFoundError so a Fireworks deployment hiccup no longer fails CI for unrelated PRs. 4. tests/llm_translation/test_gemini.py::test_gemini_image_size_limit_exceeded The test fetches the 32MB "Blue Marble 2002" image from Wikimedia to exercise the 50MB image-size cap. CI runners share an IP pool with noisy traffic, so Wikimedia routinely returns HTTP 429. The size-limit check never gets a chance to fire. Catch the 429 BadRequestError and pytest.skip in that case. None of these belong on this PR conceptually, but they're included per request to unblock the workflow before morning. --- .../test_fireworks_ai_translation.py | 23 +++++++------ tests/llm_translation/test_gemini.py | 33 +++++++++++++++---- tests/local_testing/test_completion.py | 2 ++ tests/local_testing/test_completion_cost.py | 5 ++- 4 files changed, 45 insertions(+), 18 deletions(-) diff --git a/tests/llm_translation/test_fireworks_ai_translation.py b/tests/llm_translation/test_fireworks_ai_translation.py index 1cc6aabdca..164c27ca25 100644 --- a/tests/llm_translation/test_fireworks_ai_translation.py +++ b/tests/llm_translation/test_fireworks_ai_translation.py @@ -118,16 +118,19 @@ def test_document_inlining_example(disable_add_transform_inline_image_block): disable_add_transform_inline_image_block=disable_add_transform_inline_image_block, ) else: - completion = litellm.completion( - model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct", - messages=[ - { - "role": "user", - "content": "this is a test request, write a short poem", - }, - ], - disable_add_transform_inline_image_block=disable_add_transform_inline_image_block, - ) + try: + completion = litellm.completion( + model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct", + messages=[ + { + "role": "user", + "content": "this is a test request, write a short poem", + }, + ], + disable_add_transform_inline_image_block=disable_add_transform_inline_image_block, + ) + except litellm.NotFoundError as e: + pytest.skip(f"Fireworks model unavailable upstream (404): {e}") print(completion) diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index 97b0aaee86..85039372af 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -1362,8 +1362,12 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults(): ) # For Gemini 3, should not force thinkingLevel by default - assert "thinkingLevel" not in result, "Should not force thinkingLevel for Gemini 3" - assert "thinkingBudget" not in result, "Should NOT have thinkingBudget for Gemini 3" + assert ( + "thinkingLevel" not in result + ), "Should not force thinkingLevel for Gemini 3" + assert ( + "thinkingBudget" not in result + ), "Should NOT have thinkingBudget for Gemini 3" assert result["includeThoughts"] is True # Test 2: Anthropic thinking disabled for Gemini 3 @@ -1395,7 +1399,10 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults(): ) assert result_zero["includeThoughts"] is False - assert "thinkingLevel" not in result_zero or result_zero.get("thinkingLevel") is None + assert ( + "thinkingLevel" not in result_zero + or result_zero.get("thinkingLevel") is None + ) # Test 4: Gemini 3 flash-preview should also follow provider defaults by default result_gemini3flashpreview = VertexGeminiConfig._map_thinking_param( @@ -1525,8 +1532,12 @@ def test_anthropic_thinking_param_via_map_openai_params(): # Check that thinkingConfig was created without forced thinkingLevel assert "thinkingConfig" in result, "Should have thinkingConfig in optional_params" thinking_config = result["thinkingConfig"] - assert "thinkingLevel" not in thinking_config, "Should not force thinkingLevel for Gemini 3 by default" - assert "thinkingBudget" not in thinking_config, "Should NOT have thinkingBudget for Gemini 3" + assert ( + "thinkingLevel" not in thinking_config + ), "Should not force thinkingLevel for Gemini 3 by default" + assert ( + "thinkingBudget" not in thinking_config + ), "Should NOT have thinkingBudget for Gemini 3" assert thinking_config["includeThoughts"] is True # Test with Gemini 2 model @@ -1614,8 +1625,16 @@ def test_gemini_image_size_limit_exceeded(): } ] - with pytest.raises(litellm.ImageFetchError) as excinfo: - completion(model="gemini/gemini-2.5-flash-lite", messages=messages) + try: + with pytest.raises(litellm.ImageFetchError) as excinfo: + completion(model="gemini/gemini-2.5-flash-lite", messages=messages) + except litellm.BadRequestError as e: + # Wikimedia rate-limits CI runners (HTTP 429) for the Blue Marble + # image, so the size-limit check never gets a chance to fire. Skip + # rather than fail when the upstream host blocks us. + if "429" in str(e) or "Too Many Requests" in str(e): + pytest.skip(f"Wikimedia rate-limited the test fixture image: {e}") + raise error_message = str(excinfo.value) assert "Image size" in error_message diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 6341fa7800..b92251d2d4 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -1061,6 +1061,8 @@ def test_completion_fireworks_ai(): messages=messages, ) print(response) + except litellm.NotFoundError as e: + pytest.skip(f"Fireworks model unavailable upstream (404): {e}") except Exception as e: pytest.fail(f"Error occurred: {e}") diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py index 618287e195..d62e4fbd1f 100644 --- a/tests/local_testing/test_completion_cost.py +++ b/tests/local_testing/test_completion_cost.py @@ -1190,7 +1190,10 @@ def test_completion_cost_fireworks_ai(model): litellm.model_cost = litellm.get_model_cost_map(url="") messages = [{"role": "user", "content": "Hey, how's it going?"}] - resp = litellm.completion(model=model, messages=messages) # works fine + try: + resp = litellm.completion(model=model, messages=messages) + except litellm.NotFoundError as e: + pytest.skip(f"Fireworks model unavailable upstream (404): {e}") print(resp) cost = completion_cost(completion_response=resp) From 18c932210a4dfc08936a8f6195b1d108d331a1aa Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 16 May 2026 07:54:57 +0000 Subject: [PATCH 08/10] fix(test_gemini): skip after pytest.raises catches the 429-wrapped ImageFetchError litellm.ImageFetchError is a subclass of BadRequestError, so when Wikimedia returns 429 the pytest.raises(ImageFetchError) block matches and swallows the exception -- the outer try/except never fires. Drop the try/except and check the captured error message for "Status code: 429" after the raises block, calling pytest.skip in that case. Same intent, right control flow. --- tests/llm_translation/test_gemini.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index 85039372af..b6a9b76308 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -1625,18 +1625,17 @@ def test_gemini_image_size_limit_exceeded(): } ] - try: - with pytest.raises(litellm.ImageFetchError) as excinfo: - completion(model="gemini/gemini-2.5-flash-lite", messages=messages) - except litellm.BadRequestError as e: - # Wikimedia rate-limits CI runners (HTTP 429) for the Blue Marble - # image, so the size-limit check never gets a chance to fire. Skip - # rather than fail when the upstream host blocks us. - if "429" in str(e) or "Too Many Requests" in str(e): - pytest.skip(f"Wikimedia rate-limited the test fixture image: {e}") - raise + with pytest.raises(litellm.ImageFetchError) as excinfo: + completion(model="gemini/gemini-2.5-flash-lite", messages=messages) error_message = str(excinfo.value) + # Wikimedia rate-limits CI runners (HTTP 429) for the Blue Marble image, + # so the size-limit check never gets a chance to fire and we instead + # capture the 429-wrapped ImageFetchError. Skip rather than fail when + # the upstream host blocks us -- the intent here is to exercise the + # size cap, not to test against Wikimedia availability. + if "Status code: 429" in error_message or "Too Many Requests" in error_message: + pytest.skip(f"Wikimedia rate-limited the test fixture image: {error_message}") assert "Image size" in error_message assert "exceeds maximum allowed size" in error_message From fb7091ef799775ebae995cd92f932dfa3ebf4302 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 16 May 2026 15:11:36 +0000 Subject: [PATCH 09/10] refactor(reasoning_effort_grid): tighten test helpers per Greptile review Two P2 nits flagged by Greptile on PR 28036: 1. _build_completion_kwargs() defaulted vertex_project to "vertex-check-481318" when VERTEX_PROJECT was unset. That value is a specific GCP project that doesn't belong to this repo, so if the env-var skip guard were ever bypassed (misconfig, direct helper call), the test would silently issue calls to a foreign project rather than failing loudly. Drop the fallback and read os.environ["VERTEX_PROJECT"] directly, mirroring how AZURE_FOUNDRY_* are handled. 2. _build_messages_kwargs() was a one-liner that returned the result of _build_completion_kwargs() unchanged -- a dead abstraction with one caller. Inline at the _call_messages call site and delete the helper. --- .../test_reasoning_effort_grid.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py index a815eb861e..4f2b6f0955 100644 --- a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py +++ b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py @@ -65,20 +65,13 @@ def _build_completion_kwargs(model: ModelEntry, effort: str) -> Dict[str, Any]: if effort != "__omit__": kwargs["reasoning_effort"] = effort if model.model.startswith("vertex_ai/"): - kwargs["vertex_project"] = os.environ.get( - "VERTEX_PROJECT", "vertex-check-481318" - ) + kwargs["vertex_project"] = os.environ["VERTEX_PROJECT"] if model.model.startswith("azure_ai/"): kwargs["api_base"] = os.environ["AZURE_FOUNDRY_API_BASE"] kwargs["api_key"] = os.environ["AZURE_FOUNDRY_API_KEY"] return kwargs -def _build_messages_kwargs(model: ModelEntry, effort: str) -> Dict[str, Any]: - kwargs = _build_completion_kwargs(model, effort) - return kwargs - - def _converse_subbody(body: Dict[str, Any]) -> Dict[str, Any]: """Return the dict that holds thinking/output_config for a Converse wire body.""" return body.get("additionalModelRequestFields", body) @@ -185,7 +178,7 @@ async def _call_chat(model: ModelEntry, effort: str) -> Tuple[int, Optional[Exce async def _call_messages( model: ModelEntry, effort: str ) -> Tuple[int, Optional[Exception]]: - kwargs = _build_messages_kwargs(model, effort) + kwargs = _build_completion_kwargs(model, effort) try: await litellm.anthropic_messages(**kwargs) return 200, None From f9485f1bf6d9e62498673bb3e0ce0fdd1af355f4 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 16 May 2026 15:33:29 +0000 Subject: [PATCH 10/10] refactor: strip PR-introduced docstrings and explanatory comments --- .../reasoning_effort_grid/conftest.py | 17 --------- .../reasoning_effort_grid/grid_spec.py | 15 -------- .../test_reasoning_effort_grid.py | 38 ------------------- tests/llm_translation/test_gemini.py | 5 --- .../interactions/test_openapi_compliance.py | 5 --- 5 files changed, 80 deletions(-) diff --git a/tests/llm_translation/reasoning_effort_grid/conftest.py b/tests/llm_translation/reasoning_effort_grid/conftest.py index aad85307cc..4ea2cd1d9b 100644 --- a/tests/llm_translation/reasoning_effort_grid/conftest.py +++ b/tests/llm_translation/reasoning_effort_grid/conftest.py @@ -1,12 +1,3 @@ -"""Shared fixtures for the reasoning_effort grid e2e suite. - -VCR wiring (Redis-backed cassette persister, auto-application of -``@pytest.mark.vcr`` to every collected item, cassette-cache health summary) -is inherited from ``tests/llm_translation/conftest.py``. This file only -contributes the ``wire_capture`` fixture, which records the wire body -LiteLLM sends upstream so each cell can inspect it. -""" - from typing import Any, Dict, List, Optional import pytest @@ -16,14 +7,6 @@ from litellm.integrations.custom_logger import CustomLogger class _WireBodyCapture(CustomLogger): - """Pre-call hook that records the outgoing wire body LiteLLM sends upstream. - - ``complete_input_dict`` is the fully transformed provider request as set - by every provider transformation in ``litellm/llms/**``. Capturing it here - means a regression anywhere in the transformation chain (strip, rename, - drop) surfaces as an assertion failure on the cell that depends on it. - """ - def __init__(self) -> None: super().__init__() self.records: List[Dict[str, Any]] = [] diff --git a/tests/llm_translation/reasoning_effort_grid/grid_spec.py b/tests/llm_translation/reasoning_effort_grid/grid_spec.py index 58cff57f7c..ed5346dad7 100644 --- a/tests/llm_translation/reasoning_effort_grid/grid_spec.py +++ b/tests/llm_translation/reasoning_effort_grid/grid_spec.py @@ -1,15 +1,3 @@ -""" -Canonical post-fix expectations for the reasoning_effort grid sweep. - -The original QA sweep on -https://github.com/BerriAI/litellm/pull/27039#issuecomment-4363363610 -covered 21 (provider x model) combos x 11 effort values (231 cells). The -follow-up PR https://github.com/BerriAI/litellm/pull/27074 closed nine bugs -surfaced by that sweep. This module encodes the post-fix expectations as a -small rule set keyed by (model_mode, effort) and per-model capability -overrides, then expands them across the model x effort matrix per route. -""" - from dataclasses import dataclass, field from typing import Dict, FrozenSet, List, Optional, Tuple @@ -19,8 +7,6 @@ OMIT = object() @dataclass(frozen=True) class CellExpectation: - """Expected post-fix behavior for a single grid cell.""" - status: int thinking_type: object output_config_effort: object = OMIT @@ -77,7 +63,6 @@ _BAD_REQUEST_EFFORTS: FrozenSet[str] = frozenset({"disabled", "invalid", ""}) def expected(model: ModelEntry, effort: str) -> CellExpectation: - """Compute the post-fix expected cell for a (model, effort) pair.""" if effort in ("__omit__", "none"): if model.mode == "budget": return CellExpectation(status=200, thinking_type=OMIT, max_tokens=8192) diff --git a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py index 4f2b6f0955..28e2e402d6 100644 --- a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py +++ b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py @@ -1,26 +1,3 @@ -""" -End-to-end regression suite for reasoning_effort mapping across the -Anthropic-backed routes covered by the original QA sweep. - -Encodes the 21 (provider x model) x 11 effort matrix (231 cells) from the -QA sweep on https://github.com/BerriAI/litellm/pull/27039#issuecomment-4363363610 -that the fix in https://github.com/BerriAI/litellm/pull/27074 was validated -against. Each cell asserts: - - - Wire body shape captured pre-call (thinking.type, output_config.effort, - thinking.budget_tokens, max_tokens) -- the regression signal for silent - drops/strips anywhere in the transformation chain. - - Status code returned by LiteLLM (200 vs BadRequestError -> 400) -- the - regression signal for clean-error vs leaked-500 mappings. - -Calls go to real provider endpoints, but the parent -``tests/llm_translation/conftest.py`` auto-applies ``@pytest.mark.vcr`` to -every collected item, so first run records cassettes (Redis-backed) and -subsequent CI runs replay them with no live spend. Each route still skips at -runtime when its required env vars are absent, so PR builds without provider -credentials no-op gracefully. -""" - import json import os from typing import Any, Dict, List, Optional, Tuple @@ -73,7 +50,6 @@ def _build_completion_kwargs(model: ModelEntry, effort: str) -> Dict[str, Any]: def _converse_subbody(body: Dict[str, Any]) -> Dict[str, Any]: - """Return the dict that holds thinking/output_config for a Converse wire body.""" return body.get("additionalModelRequestFields", body) @@ -92,7 +68,6 @@ def _assert_cell( assert status == cell.status, f"expected status={cell.status}, got status={status}" if cell.status != 200: - # Bad-request paths short-circuit before the wire body matters. return assert body is not None, "wire body was not captured for a 200-status cell" @@ -150,14 +125,6 @@ _PARAM_IDS: List[str] = [_cell_id(case) for case in _PARAMS] def _classify_status(exc: Exception) -> int: - """Map an exception to the HTTP status the QA grid would have observed. - - The Anthropic Messages route (litellm.anthropic_messages) wraps client-side - BadRequestError as ``AnthropicError`` with ``status_code=400`` rather than - re-raising the BadRequestError class directly, so isinstance() alone misses - those cells. Read the ``status_code`` attribute when present (set by every - BaseLLMException subclass) and fall through to 500 otherwise. - """ if isinstance(exc, BadRequestError): return 400 code = getattr(exc, "status_code", None) @@ -208,9 +175,6 @@ async def test_reasoning_effort_grid( record = wire_capture.latest() body = record["body"] if record else None - # Bedrock Converse logs `complete_input_dict` as a JSON string (see - # litellm/llms/bedrock/chat/converse_handler.py); parse it so the dict - # accessors in `_assert_cell` work uniformly across routes. if route_name == "bedrock_converse" and isinstance(body, str): body = json.loads(body) @@ -225,7 +189,6 @@ async def test_reasoning_effort_grid( def test_grid_cell_count() -> None: - """Guard against accidental drops or duplicates in the grid spec.""" assert len(_PARAMS) == 21 * 11, ( f"expected 231 cells (21 provider x model combos x 11 efforts), " f"got {len(_PARAMS)}" @@ -233,7 +196,6 @@ def test_grid_cell_count() -> None: def test_grid_route_coverage() -> None: - """The grid must cover every route the original QA sweep covered.""" route_names = {route.name for route in ROUTES} assert route_names == { "anthropic_direct", diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index b6a9b76308..a02a1ffe60 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -1629,11 +1629,6 @@ def test_gemini_image_size_limit_exceeded(): completion(model="gemini/gemini-2.5-flash-lite", messages=messages) error_message = str(excinfo.value) - # Wikimedia rate-limits CI runners (HTTP 429) for the Blue Marble image, - # so the size-limit check never gets a chance to fire and we instead - # capture the 429-wrapped ImageFetchError. Skip rather than fail when - # the upstream host blocks us -- the intent here is to exercise the - # size cap, not to test against Wikimedia availability. if "Status code: 429" in error_message or "Too Many Requests" in error_message: pytest.skip(f"Wikimedia rate-limited the test fixture image: {error_message}") assert "Image size" in error_message diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index ce74bf150a..a89e4c9dd5 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -157,11 +157,6 @@ class TestResponseCompliance: # Check CreateModelInteractionParams which includes output fields schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"] - # Output fields (readOnly). Google's live spec has churned through - # both `outputs` and `steps` for the per-turn output array and at the - # moment carries neither -- only the stable response-level fields - # below are guaranteed. Re-add the per-turn key once upstream - # stabilizes on a name. output_fields = [ "id", "status",