Merge pull request #28036 from BerriAI/litellm_grid-v4-e2e-tests-cZRwz

test(ci): add reasoning_effort grid e2e regression suite

tests/llm_translation/reasoning_effort_grid/conftest.py

@@ -0,0 +1,38 @@
+from typing import Any, Dict, List, Optional
+
+import pytest
+
+import litellm
+from litellm.integrations.custom_logger import CustomLogger
+
+
+class _WireBodyCapture(CustomLogger):
+    def __init__(self) -> None:
+        super().__init__()
+        self.records: List[Dict[str, Any]] = []
+
+    def log_pre_api_call(self, model, messages, kwargs):
+        self.records.append(
+            {
+                "model": model,
+                "body": kwargs.get("additional_args", {}).get("complete_input_dict"),
+                "api_base": kwargs.get("additional_args", {}).get("api_base"),
+            }
+        )
+
+    async def async_log_pre_api_call(self, model, messages, kwargs):
+        self.log_pre_api_call(model, messages, kwargs)
+
+    def latest(self) -> Optional[Dict[str, Any]]:
+        return self.records[-1] if self.records else None
+
+
+@pytest.fixture()
+def wire_capture():
+    capture = _WireBodyCapture()
+    previous = list(litellm.callbacks)
+    litellm.callbacks = previous + [capture]
+    try:
+        yield capture
+    finally:
+        litellm.callbacks = previous

tests/llm_translation/reasoning_effort_grid/grid_spec.py

@@ -0,0 +1,289 @@
+from dataclasses import dataclass, field
+from typing import Dict, FrozenSet, List, Optional, Tuple
+
+
+OMIT = object()
+
+
+@dataclass(frozen=True)
+class CellExpectation:
+    status: int
+    thinking_type: object
+    output_config_effort: object = OMIT
+    thinking_budget_tokens: object = OMIT
+    max_tokens: object = OMIT
+
+
+@dataclass(frozen=True)
+class ModelEntry:
+    alias: str
+    model: str
+    mode: str
+    extra_params: Tuple[Tuple[str, str], ...] = field(default_factory=tuple)
+    required_env: FrozenSet[str] = field(default_factory=frozenset)
+    caps: FrozenSet[str] = field(default_factory=frozenset)
+
+    def params(self) -> Dict[str, str]:
+        return dict(self.extra_params)
+
+
+EFFORTS: Tuple[str, ...] = (
+    "__omit__",
+    "none",
+    "minimal",
+    "low",
+    "medium",
+    "high",
+    "xhigh",
+    "max",
+    "disabled",
+    "invalid",
+    "",
+)
+
+_BUDGET_TOKENS: Dict[str, int] = {
+    "minimal": 1024,
+    "low": 1024,
+    "medium": 2048,
+    "high": 4096,
+    "xhigh": 8192,
+    "max": 16384,
+}
+
+_ADAPTIVE_EFFORT_LABEL: Dict[str, str] = {
+    "minimal": "low",
+    "low": "low",
+    "medium": "medium",
+    "high": "high",
+    "xhigh": "xhigh",
+    "max": "max",
+}
+
+_BAD_REQUEST_EFFORTS: FrozenSet[str] = frozenset({"disabled", "invalid", ""})
+
+
+def expected(model: ModelEntry, effort: str) -> CellExpectation:
+    if effort in ("__omit__", "none"):
+        if model.mode == "budget":
+            return CellExpectation(status=200, thinking_type=OMIT, max_tokens=8192)
+        return CellExpectation(status=200, thinking_type=OMIT)
+
+    if effort in _BAD_REQUEST_EFFORTS:
+        return CellExpectation(status=400, thinking_type=OMIT)
+
+    if effort in ("xhigh", "max"):
+        cap = f"supports_{effort}_reasoning_effort"
+        if cap not in model.caps:
+            return CellExpectation(status=400, thinking_type=OMIT)
+
+    if model.mode == "adaptive":
+        return CellExpectation(
+            status=200,
+            thinking_type="adaptive",
+            output_config_effort=_ADAPTIVE_EFFORT_LABEL[effort],
+        )
+
+    return CellExpectation(
+        status=200,
+        thinking_type="enabled",
+        thinking_budget_tokens=_BUDGET_TOKENS[effort],
+        max_tokens=8192,
+    )
+
+
+_ANTHROPIC_REQ = frozenset({"ANTHROPIC_API_KEY"})
+_AZURE_FOUNDRY_REQ = frozenset({"AZURE_FOUNDRY_API_BASE", "AZURE_FOUNDRY_API_KEY"})
+_VERTEX_REQ = frozenset({"VERTEX_PROJECT"})
+_BEDROCK_REQ = frozenset({"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"})
+
+
+_CAPS_OPUS_4_7: FrozenSet[str] = frozenset(
+    {"supports_xhigh_reasoning_effort", "supports_max_reasoning_effort"}
+)
+_CAPS_4_6: FrozenSet[str] = frozenset({"supports_max_reasoning_effort"})
+_CAPS_NONE: FrozenSet[str] = frozenset()
+
+
+ANTHROPIC_DIRECT_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="claude-opus-4-7",
+        model="anthropic/claude-opus-4-7",
+        mode="adaptive",
+        required_env=_ANTHROPIC_REQ,
+        caps=_CAPS_OPUS_4_7,
+    ),
+    ModelEntry(
+        alias="claude-sonnet-4-6",
+        model="anthropic/claude-sonnet-4-6",
+        mode="adaptive",
+        required_env=_ANTHROPIC_REQ,
+        caps=_CAPS_4_6,
+    ),
+    ModelEntry(
+        alias="claude-haiku-4-5",
+        model="anthropic/claude-haiku-4-5",
+        mode="budget",
+        required_env=_ANTHROPIC_REQ,
+        caps=_CAPS_NONE,
+    ),
+)
+
+
+AZURE_AI_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="azure-claude-opus-4-7",
+        model="azure_ai/claude-opus-4-7",
+        mode="adaptive",
+        required_env=_AZURE_FOUNDRY_REQ,
+        caps=_CAPS_OPUS_4_7,
+    ),
+    ModelEntry(
+        alias="azure-claude-opus-4-6",
+        model="azure_ai/claude-opus-4-6",
+        mode="adaptive",
+        required_env=_AZURE_FOUNDRY_REQ,
+        caps=_CAPS_4_6,
+    ),
+    ModelEntry(
+        alias="azure-claude-sonnet-4-6",
+        model="azure_ai/claude-sonnet-4-6",
+        mode="adaptive",
+        required_env=_AZURE_FOUNDRY_REQ,
+        caps=_CAPS_4_6,
+    ),
+    ModelEntry(
+        alias="azure-claude-haiku-4-5",
+        model="azure_ai/claude-haiku-4-5",
+        mode="budget",
+        required_env=_AZURE_FOUNDRY_REQ,
+        caps=_CAPS_NONE,
+    ),
+)
+
+
+VERTEX_AI_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="vertex-claude-opus-4-7",
+        model="vertex_ai/claude-opus-4-7",
+        mode="adaptive",
+        extra_params=(("vertex_location", "global"),),
+        required_env=_VERTEX_REQ,
+        caps=_CAPS_OPUS_4_7,
+    ),
+    ModelEntry(
+        alias="vertex-claude-opus-4-6",
+        model="vertex_ai/claude-opus-4-6",
+        mode="adaptive",
+        extra_params=(("vertex_location", "us-east5"),),
+        required_env=_VERTEX_REQ,
+        caps=_CAPS_4_6,
+    ),
+    ModelEntry(
+        alias="vertex-claude-sonnet-4-6",
+        model="vertex_ai/claude-sonnet-4-6",
+        mode="adaptive",
+        extra_params=(("vertex_location", "us-east5"),),
+        required_env=_VERTEX_REQ,
+        caps=_CAPS_4_6,
+    ),
+    ModelEntry(
+        alias="vertex-claude-haiku-4-5",
+        model="vertex_ai/claude-haiku-4-5",
+        mode="budget",
+        extra_params=(("vertex_location", "us-east5"),),
+        required_env=_VERTEX_REQ,
+        caps=_CAPS_NONE,
+    ),
+)
+
+
+BEDROCK_CONVERSE_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="bedrock-claude-opus-4-7",
+        model="bedrock/converse/us.anthropic.claude-opus-4-7",
+        mode="adaptive",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_OPUS_4_7,
+    ),
+    ModelEntry(
+        alias="bedrock-claude-opus-4-6",
+        model="bedrock/converse/us.anthropic.claude-opus-4-6-v1",
+        mode="adaptive",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_4_6,
+    ),
+    ModelEntry(
+        alias="bedrock-claude-sonnet-4-6",
+        model="bedrock/converse/us.anthropic.claude-sonnet-4-6",
+        mode="adaptive",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_4_6,
+    ),
+    ModelEntry(
+        alias="bedrock-claude-sonnet-4-5",
+        model="bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
+        mode="budget",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_NONE,
+    ),
+)
+
+
+BEDROCK_INVOKE_CHAT_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="bedrock-invoke-claude-opus-4-6",
+        model="bedrock/invoke/us.anthropic.claude-opus-4-6-v1",
+        mode="adaptive",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_4_6,
+    ),
+    ModelEntry(
+        alias="bedrock-invoke-claude-sonnet-4-6",
+        model="bedrock/invoke/us.anthropic.claude-sonnet-4-6",
+        mode="adaptive",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_4_6,
+    ),
+    ModelEntry(
+        alias="bedrock-invoke-claude-opus-4-5",
+        model="bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0",
+        mode="budget",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_NONE,
+    ),
+)
+
+
+BEDROCK_INVOKE_MESSAGES_MODELS: Tuple[ModelEntry, ...] = BEDROCK_INVOKE_CHAT_MODELS
+
+
+@dataclass(frozen=True)
+class Route:
+    name: str
+    models: Tuple[ModelEntry, ...]
+
+
+ROUTES: Tuple[Route, ...] = (
+    Route("anthropic_direct", ANTHROPIC_DIRECT_MODELS),
+    Route("azure_ai", AZURE_AI_MODELS),
+    Route("vertex_ai", VERTEX_AI_MODELS),
+    Route("bedrock_converse", BEDROCK_CONVERSE_MODELS),
+    Route("bedrock_invoke_chat", BEDROCK_INVOKE_CHAT_MODELS),
+    Route("bedrock_invoke_messages", BEDROCK_INVOKE_MESSAGES_MODELS),
+)
+
+
+def all_cells() -> List[Tuple[str, ModelEntry, str, CellExpectation]]:
+    cells: List[Tuple[str, ModelEntry, str, CellExpectation]] = []
+    for route in ROUTES:
+        for model in route.models:
+            for effort in EFFORTS:
+                cells.append((route.name, model, effort, expected(model, effort)))
+    return cells

tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py

@@ -0,0 +1,207 @@
+import json
+import os
+from typing import Any, Dict, List, Optional, Tuple
+
+import pytest
+
+import litellm
+from litellm.exceptions import BadRequestError
+
+from .grid_spec import (
+    OMIT,
+    ROUTES,
+    CellExpectation,
+    ModelEntry,
+    all_cells,
+)
+
+
+_PROMPT_MESSAGES: List[Dict[str, str]] = [
+    {"role": "user", "content": "Step by step, calculate 47 * 53. Show your work."}
+]
+
+
+def _required_env_missing(model: ModelEntry) -> Optional[str]:
+    missing = [key for key in model.required_env if not os.environ.get(key)]
+    if missing:
+        return "missing env: " + ", ".join(sorted(missing))
+    return None
+
+
+def _max_tokens_for(model: ModelEntry) -> int:
+    return 200 if model.mode == "adaptive" else 8192
+
+
+def _build_completion_kwargs(model: ModelEntry, effort: str) -> Dict[str, Any]:
+    kwargs: Dict[str, Any] = {
+        "model": model.model,
+        "messages": _PROMPT_MESSAGES,
+        "max_tokens": _max_tokens_for(model),
+    }
+    kwargs.update(model.params())
+    if effort != "__omit__":
+        kwargs["reasoning_effort"] = effort
+    if model.model.startswith("vertex_ai/"):
+        kwargs["vertex_project"] = os.environ["VERTEX_PROJECT"]
+    if model.model.startswith("azure_ai/"):
+        kwargs["api_base"] = os.environ["AZURE_FOUNDRY_API_BASE"]
+        kwargs["api_key"] = os.environ["AZURE_FOUNDRY_API_KEY"]
+    return kwargs
+
+
+def _converse_subbody(body: Dict[str, Any]) -> Dict[str, Any]:
+    return body.get("additionalModelRequestFields", body)
+
+
+def _max_tokens_from_body(body: Dict[str, Any], route_name: str) -> Optional[int]:
+    if route_name == "bedrock_converse":
+        return body.get("inferenceConfig", {}).get("maxTokens")
+    return body.get("max_tokens")
+
+
+def _assert_cell(
+    route_name: str,
+    body: Optional[Dict[str, Any]],
+    status: int,
+    cell: CellExpectation,
+) -> None:
+    assert status == cell.status, f"expected status={cell.status}, got status={status}"
+
+    if cell.status != 200:
+        return
+
+    assert body is not None, "wire body was not captured for a 200-status cell"
+    subbody = _converse_subbody(body) if route_name == "bedrock_converse" else body
+    thinking = subbody.get("thinking")
+    output_config = subbody.get("output_config")
+
+    if cell.thinking_type is OMIT:
+        assert thinking is None, f"expected thinking omitted, got {thinking!r}"
+    else:
+        assert thinking is not None, "expected thinking present, got omit"
+        assert thinking.get("type") == cell.thinking_type, (
+            f"expected thinking.type={cell.thinking_type!r}, "
+            f"got {thinking.get('type')!r}"
+        )
+
+    if cell.output_config_effort is OMIT:
+        assert (
+            output_config is None or "effort" not in output_config
+        ), f"expected output_config.effort omitted, got {output_config!r}"
+    else:
+        assert output_config is not None, (
+            f"expected output_config.effort={cell.output_config_effort!r}, "
+            "got output_config omitted"
+        )
+        assert output_config.get("effort") == cell.output_config_effort, (
+            f"expected output_config.effort={cell.output_config_effort!r}, "
+            f"got {output_config.get('effort')!r}"
+        )
+
+    if cell.thinking_budget_tokens is not OMIT:
+        assert thinking is not None
+        assert thinking.get("budget_tokens") == cell.thinking_budget_tokens, (
+            f"expected thinking.budget_tokens={cell.thinking_budget_tokens!r}, "
+            f"got {thinking.get('budget_tokens')!r}"
+        )
+
+    if cell.max_tokens is not OMIT:
+        wire_max = _max_tokens_from_body(body, route_name)
+        assert (
+            wire_max == cell.max_tokens
+        ), f"expected max_tokens={cell.max_tokens!r}, got {wire_max!r}"
+
+
+_PARAMS: List[Tuple[str, ModelEntry, str, CellExpectation]] = all_cells()
+
+
+def _cell_id(case: Tuple[str, ModelEntry, str, CellExpectation]) -> str:
+    route_name, model, effort, _ = case
+    effort_label = "__empty__" if effort == "" else effort
+    return f"{route_name}-{model.alias}-{effort_label}"
+
+
+_PARAM_IDS: List[str] = [_cell_id(case) for case in _PARAMS]
+
+
+def _classify_status(exc: Exception) -> int:
+    if isinstance(exc, BadRequestError):
+        return 400
+    code = getattr(exc, "status_code", None)
+    if isinstance(code, int):
+        return code
+    return 500
+
+
+async def _call_chat(model: ModelEntry, effort: str) -> Tuple[int, Optional[Exception]]:
+    kwargs = _build_completion_kwargs(model, effort)
+    try:
+        await litellm.acompletion(**kwargs)
+        return 200, None
+    except Exception as exc:
+        return _classify_status(exc), exc
+
+
+async def _call_messages(
+    model: ModelEntry, effort: str
+) -> Tuple[int, Optional[Exception]]:
+    kwargs = _build_completion_kwargs(model, effort)
+    try:
+        await litellm.anthropic_messages(**kwargs)
+        return 200, None
+    except Exception as exc:
+        return _classify_status(exc), exc
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("route_name", "model", "effort", "cell"), _PARAMS, ids=_PARAM_IDS
+)
+async def test_reasoning_effort_grid(
+    route_name: str,
+    model: ModelEntry,
+    effort: str,
+    cell: CellExpectation,
+    wire_capture,
+) -> None:
+    skip_reason = _required_env_missing(model)
+    if skip_reason:
+        pytest.skip(skip_reason)
+
+    if route_name == "bedrock_invoke_messages":
+        status, exc = await _call_messages(model, effort)
+    else:
+        status, exc = await _call_chat(model, effort)
+
+    record = wire_capture.latest()
+    body = record["body"] if record else None
+    if route_name == "bedrock_converse" and isinstance(body, str):
+        body = json.loads(body)
+
+    try:
+        _assert_cell(route_name, body, status, cell)
+    except AssertionError:
+        if exc is not None:
+            raise AssertionError(
+                f"underlying exception ({type(exc).__name__}): {exc}"
+            ) from None
+        raise
+
+
+def test_grid_cell_count() -> None:
+    assert len(_PARAMS) == 21 * 11, (
+        f"expected 231 cells (21 provider x model combos x 11 efforts), "
+        f"got {len(_PARAMS)}"
+    )
+
+
+def test_grid_route_coverage() -> None:
+    route_names = {route.name for route in ROUTES}
+    assert route_names == {
+        "anthropic_direct",
+        "azure_ai",
+        "vertex_ai",
+        "bedrock_converse",
+        "bedrock_invoke_chat",
+        "bedrock_invoke_messages",
+    }

tests/llm_translation/test_gemini.py

@@ -1362,8 +1362,12 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults():
         )
 
         # For Gemini 3, should not force thinkingLevel by default
-        assert "thinkingLevel" not in result, "Should not force thinkingLevel for Gemini 3"
-        assert "thinkingBudget" not in result, "Should NOT have thinkingBudget for Gemini 3"
+        assert (
+            "thinkingLevel" not in result
+        ), "Should not force thinkingLevel for Gemini 3"
+        assert (
+            "thinkingBudget" not in result
+        ), "Should NOT have thinkingBudget for Gemini 3"
         assert result["includeThoughts"] is True
 
         # Test 2: Anthropic thinking disabled for Gemini 3
@@ -1395,7 +1399,10 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults():
         )
 
         assert result_zero["includeThoughts"] is False
-        assert "thinkingLevel" not in result_zero or result_zero.get("thinkingLevel") is None
+        assert (
+            "thinkingLevel" not in result_zero
+            or result_zero.get("thinkingLevel") is None
+        )
 
         # Test 4: Gemini 3 flash-preview should also follow provider defaults by default
         result_gemini3flashpreview = VertexGeminiConfig._map_thinking_param(
@@ -1525,8 +1532,12 @@ def test_anthropic_thinking_param_via_map_openai_params():
     # Check that thinkingConfig was created without forced thinkingLevel
     assert "thinkingConfig" in result, "Should have thinkingConfig in optional_params"
     thinking_config = result["thinkingConfig"]
-    assert "thinkingLevel" not in thinking_config, "Should not force thinkingLevel for Gemini 3 by default"
-    assert "thinkingBudget" not in thinking_config, "Should NOT have thinkingBudget for Gemini 3"
+    assert (
+        "thinkingLevel" not in thinking_config
+    ), "Should not force thinkingLevel for Gemini 3 by default"
+    assert (
+        "thinkingBudget" not in thinking_config
+    ), "Should NOT have thinkingBudget for Gemini 3"
     assert thinking_config["includeThoughts"] is True
 
     # Test with Gemini 2 model