fix(ci): make litellm_internal_staging green (logging test + Bedrock Opus 4.7 self-heal) (#29344)
* test(logging): align DB metrics event_metadata assertions with safe redaction PR #28909 hardened log_db_metrics to emit a minimal, non-sensitive event_metadata (only table_name when present, otherwise None) instead of dumping function_name, function_kwargs, and function_args onto the span. The test in test_log_db_redis_services was not updated and still asserted "function_name" in event_metadata, which raised TypeError (argument of type 'NoneType' is not iterable) and turned the logging_testing CI job red on litellm_internal_staging. Update test_log_db_metrics_success to assert event_metadata is None when no table_name is passed, and add test_log_db_metrics_event_metadata_is_safe as a regression guard verifying that only the table name surfaces and that sensitive kwargs (tokens, prisma client) are never dumped. * test(bedrock): self-heal opus-4-7 grid cells when unentitled on CI The bedrock-claude-opus-4-7 converse cells are unentitled on the Bedrock CI account, so they were marked xfail. xfail keeps reporting them as expected failures even after access is granted, so the wire translation never gets verified again. Now the cell makes the call and skips only when Bedrock replies "is not available for this account"; the moment the model is entitled the same cells run their full assertions with no edit. A focused unit test pins the tolerance predicate so any other failure still surfaces loudly and the available path still runs the assertions.
This commit is contained in:
parent
2d4c13c00f
commit
bfbb5d2375
@ -1,7 +1,6 @@
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, FrozenSet, List, Optional, Tuple
|
||||
|
||||
|
||||
OMIT = object()
|
||||
|
||||
|
||||
@ -22,6 +21,7 @@ class ModelEntry:
|
||||
extra_params: Tuple[Tuple[str, str], ...] = field(default_factory=tuple)
|
||||
required_env: FrozenSet[str] = field(default_factory=frozenset)
|
||||
caps: FrozenSet[str] = field(default_factory=frozenset)
|
||||
unavailable_error: Optional[str] = None
|
||||
bedrock_effort_ceiling: Optional[str] = None
|
||||
|
||||
def params(self) -> Dict[str, str]:
|
||||
@ -234,6 +234,7 @@ BEDROCK_CONVERSE_MODELS: Tuple[ModelEntry, ...] = (
|
||||
extra_params=(("aws_region_name", "us-east-1"),),
|
||||
required_env=_BEDROCK_REQ,
|
||||
caps=_CAPS_OPUS_4_7,
|
||||
unavailable_error="is not available for this account",
|
||||
),
|
||||
ModelEntry(
|
||||
alias="bedrock-claude-opus-4-6",
|
||||
|
||||
@ -15,7 +15,6 @@ from .grid_spec import (
|
||||
all_cells,
|
||||
)
|
||||
|
||||
|
||||
_PROMPT_MESSAGES: List[Dict[str, str]] = [
|
||||
{"role": "user", "content": "Step by step, calculate 47 * 53. Show your work."}
|
||||
]
|
||||
@ -133,6 +132,12 @@ def _classify_status(exc: Exception) -> int:
|
||||
return 500
|
||||
|
||||
|
||||
def _model_unavailable(model: ModelEntry, exc: Optional[Exception]) -> bool:
|
||||
if not model.unavailable_error or exc is None:
|
||||
return False
|
||||
return model.unavailable_error in str(exc)
|
||||
|
||||
|
||||
async def _call_chat(model: ModelEntry, effort: str) -> Tuple[int, Optional[Exception]]:
|
||||
kwargs = _build_completion_kwargs(model, effort)
|
||||
try:
|
||||
@ -173,6 +178,9 @@ async def test_reasoning_effort_grid(
|
||||
else:
|
||||
status, exc = await _call_chat(model, effort)
|
||||
|
||||
if _model_unavailable(model, exc):
|
||||
pytest.skip(f"{model.alias}: {model.unavailable_error}")
|
||||
|
||||
record = wire_capture.latest()
|
||||
body = record["body"] if record else None
|
||||
if route_name == "bedrock_converse" and isinstance(body, str):
|
||||
@ -205,3 +213,30 @@ def test_grid_route_coverage() -> None:
|
||||
"bedrock_invoke_chat",
|
||||
"bedrock_invoke_messages",
|
||||
}
|
||||
|
||||
|
||||
def test_model_unavailable_tolerates_only_the_declared_error() -> None:
|
||||
gated = ModelEntry(
|
||||
alias="bedrock-claude-opus-4-7",
|
||||
model="bedrock/converse/us.anthropic.claude-opus-4-7",
|
||||
mode="adaptive",
|
||||
unavailable_error="is not available for this account",
|
||||
)
|
||||
entitlement_error = Exception(
|
||||
"litellm.APIConnectionError: BedrockException - "
|
||||
'{"message":"anthropic.claude-opus-4-7 is not available for this account."}'
|
||||
)
|
||||
|
||||
assert _model_unavailable(gated, entitlement_error) is True
|
||||
assert (
|
||||
_model_unavailable(gated, Exception("ThrottlingException: rate exceeded"))
|
||||
is False
|
||||
)
|
||||
assert _model_unavailable(gated, None) is False
|
||||
|
||||
ungated = ModelEntry(
|
||||
alias="bedrock-claude-opus-4-6",
|
||||
model="bedrock/converse/us.anthropic.claude-opus-4-6-v1",
|
||||
mode="adaptive",
|
||||
)
|
||||
assert _model_unavailable(ungated, entitlement_error) is False
|
||||
|
||||
@ -2,7 +2,6 @@ import io
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../.."))
|
||||
|
||||
import asyncio
|
||||
@ -59,7 +58,38 @@ async def test_log_db_metrics_success():
|
||||
assert isinstance(call_args["duration"], float)
|
||||
assert isinstance(call_args["start_time"], datetime)
|
||||
assert isinstance(call_args["end_time"], datetime)
|
||||
assert "function_name" in call_args["event_metadata"]
|
||||
assert call_args["event_metadata"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_log_db_metrics_event_metadata_is_safe():
|
||||
"""event_metadata must surface only the table name, never the raw
|
||||
kwargs/args which carry live clients (Prisma, OTel spans) and secrets.
|
||||
|
||||
Regression guard for #28909: a previous version dumped function_kwargs and
|
||||
function_args onto the span.
|
||||
"""
|
||||
with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging:
|
||||
mock_proxy_logging.service_logging_obj.async_service_success_hook = AsyncMock()
|
||||
|
||||
@log_db_metrics
|
||||
async def db_call(**kwargs):
|
||||
return "success"
|
||||
|
||||
await db_call(
|
||||
parent_otel_span="test_span",
|
||||
table_name="LiteLLM_SpendLogs",
|
||||
token="sk-secret-should-not-leak",
|
||||
prisma_client=object(),
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
call_args = (
|
||||
mock_proxy_logging.service_logging_obj.async_service_success_hook.call_args[
|
||||
1
|
||||
]
|
||||
)
|
||||
assert call_args["event_metadata"] == {"table_name": "LiteLLM_SpendLogs"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Loading…
Reference in New Issue
Block a user