diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 6c8510380a..81fdc5a1e2 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -702,6 +702,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): }, ) + # _record_exception_on_span only stamps when error_code is set; + # bare TypeError etc. has none, and the span is about to be ended. + error_code = ( + error_information.get("error_code") if error_information else None + ) + if not error_code: + self.set_response_status_code_attribute(parent_otel_span, 500) + # Pre-request latency (request_data carries the propagated # metadata on the failure path; omitted if it failed before handoff). self.set_preprocessing_duration_attribute(parent_otel_span, request_data) @@ -798,11 +806,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # Pre-request latency on the SERVER span (success path). self.set_preprocessing_duration_attribute(parent_span, kwargs) - # http.response.status_code on the SERVER span (success path). - # A successful proxy response is HTTP 200; the failure path sets - # this from the error code in _record_exception_on_span. - self.set_response_status_code_attribute(parent_span, 200) - # 3. Guardrail span self._create_guardrail_span(kwargs=kwargs, context=ctx) @@ -985,7 +988,15 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): and hasattr(proxy_span, "is_recording") and proxy_span.is_recording() ): - proxy_span.end(end_time=self._to_ns(end_time)) + self._close_proxy_span_ok(proxy_span, end_time) + + def _close_proxy_span_ok(self, span: Span, end_time) -> None: + """Stamp http.response.status_code=200 + status=OK, then end the span.""" + from opentelemetry.trace import Status, StatusCode + + self.set_response_status_code_attribute(span, 200) + span.set_status(Status(StatusCode.OK)) + span.end(end_time=self._to_ns(end_time)) def _handle_success(self, kwargs, response_obj, start_time, end_time): """Create the litellm_request span then close the proxy span.""" @@ -1071,8 +1082,10 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): parent_span is not None and hasattr(parent_span, "name") and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME + and hasattr(parent_span, "is_recording") + and parent_span.is_recording() ): - parent_span.end(end_time=self._to_ns(end_time)) + self._close_proxy_span_ok(parent_span, end_time) # Stamp team attributes onto the SERVER (root) span before it is # closed, so the trace root carries them like every child span. @@ -3041,6 +3054,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): management_endpoint_span.set_status(Status(StatusCode.OK)) management_endpoint_span.end(end_time=_end_time_ns) + # The management wrapper has no other hook that closes the SERVER span. + self.set_response_status_code_attribute(parent_otel_span, 200) + parent_otel_span.set_status(Status(StatusCode.OK)) + parent_otel_span.end(end_time=_end_time_ns) + async def async_management_endpoint_failure_hook( self, logging_payload: ManagementEndpointLoggingPayload, @@ -3091,6 +3109,24 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): management_endpoint_span.set_status(Status(StatusCode.ERROR)) management_endpoint_span.end(end_time=_end_time_ns) + # The management wrapper has no other hook that closes the SERVER span. + from litellm.litellm_core_utils.litellm_logging import ( + StandardLoggingPayloadSetup, + ) + + error_information = StandardLoggingPayloadSetup.get_error_information( + original_exception=_exception, + ) + parent_otel_span.set_status(Status(StatusCode.ERROR)) + self._record_exception_on_span( + span=parent_otel_span, + kwargs={ + "exception": _exception, + "standard_logging_object": {"error_information": error_information}, + }, + ) + parent_otel_span.end(end_time=_end_time_ns) + def create_litellm_proxy_request_started_span( self, start_time: datetime, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 2ab037afb0..63fa0e6469 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -5140,13 +5140,17 @@ class StandardLoggingPayloadSetup: ) -> StandardLoggingPayloadErrorInformation: from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG - # Check for 'code' first (used by ProxyException), then fall back to 'status_code' (used by LiteLLM exceptions) - # Ensure error_code is always a string for Prisma Python JSON field compatibility + # ProxyException uses .code, LiteLLM exceptions use .status_code, + # httpx.HTTPStatusError exposes status only as .response.status_code. + # Stringified for Prisma JSON compatibility. error_code_attr = getattr(original_exception, "code", None) if error_code_attr is not None and str(error_code_attr) not in ("", "None"): error_status: str = str(error_code_attr) else: status_code_attr = getattr(original_exception, "status_code", None) + if status_code_attr is None: + response_attr = getattr(original_exception, "response", None) + status_code_attr = getattr(response_attr, "status_code", None) error_status = str(status_code_attr) if status_code_attr is not None else "" error_class: str = ( str(original_exception.__class__.__name__) if original_exception else "" diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 6974860a22..0327863392 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -671,6 +671,37 @@ async def _resolve_jwt_to_virtual_key( return None +def _ensure_parent_otel_span_on_request_state(request: Request) -> None: + """Idempotently create the OTEL SERVER span and stash it on + ``request.state.parent_otel_span``. Safe to call multiple times. + + Called both at the top of ``user_api_key_auth`` (so body-parse failures + have a span to close) and inside ``_user_api_key_auth_builder`` (for + callers that bypass ``user_api_key_auth``, e.g. MCP). + """ + from litellm.proxy.proxy_server import open_telemetry_logger + + if open_telemetry_logger is None: + return + if getattr(request.state, "parent_otel_span", None) is not None: + return + start_time = datetime.now() + try: + request.state.litellm_received_at = start_time + except Exception: + pass + parent_otel_span = open_telemetry_logger.create_litellm_proxy_request_started_span( + start_time=start_time, + headers=_safe_get_request_headers(request), + ) + open_telemetry_logger.set_proxy_request_route_attributes( + parent_otel_span, + url_path=get_request_route(request=request), + http_route=get_request_route_template(request), + ) + request.state.parent_otel_span = parent_otel_span + + async def _user_api_key_auth_builder( # noqa: PLR0915 request: Request, api_key: str, @@ -697,9 +728,10 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ) parent_otel_span: Optional[Span] = None - start_time = datetime.now() - # Stash the proxy-receive instant for the pre-request latency calc — - # the OTel Span API exposes no start-time getter, so propagate it. + # Prefer the receive-instant stamped by the early helper in + # user_api_key_auth (before body parse) — overwriting it would shorten + # the preprocessing-duration measurement by the body-parse window. + start_time = getattr(request.state, "litellm_received_at", None) or datetime.now() try: request.state.litellm_received_at = start_time except Exception: @@ -739,18 +771,11 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ) if open_telemetry_logger is not None: - parent_otel_span = ( - open_telemetry_logger.create_litellm_proxy_request_started_span( - start_time=start_time, - headers=_safe_get_request_headers(request), - ) - ) - # `route` is the literal path; template from the matched route. - open_telemetry_logger.set_proxy_request_route_attributes( - parent_otel_span, - url_path=route, - http_route=get_request_route_template(request), - ) + # Reuse the span created by user_api_key_auth (before body parse) + # so it survives _read_request_body failures. For callers that + # bypass user_api_key_auth (e.g. MCP), create it lazily. + _ensure_parent_otel_span_on_request_state(request) + parent_otel_span = getattr(request.state, "parent_otel_span", None) ### USER-DEFINED AUTH FUNCTION ### if enterprise_custom_auth is not None: @@ -2149,6 +2174,12 @@ async def user_api_key_auth( Parent function to authenticate user api key / jwt token. """ + # Create the SERVER span and stash it on request.state BEFORE reading the + # body. _read_request_body can raise ProxyException for malformed JSON; + # without this, that path leaves no span for the exception handler to + # close, and the trace never reaches the backend. + _ensure_parent_otel_span_on_request_state(request) + request_data = await _read_request_body(request=request) request_data = populate_request_with_path_params( request_data=request_data, request=request diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index eebcf0b657..a67d8d934b 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -3226,6 +3226,7 @@ async def info_key_fn_v2( @router.get( "/key/info", tags=["key management"], dependencies=[Depends(user_api_key_auth)] ) +@management_endpoint_wrapper async def info_key_fn( key: Optional[str] = fastapi.Query( default=None, description="Key in the request parameters" diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index f2d6e9612f..b7d5cc30c4 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -518,19 +518,23 @@ def management_endpoint_wrapper(func): _request_body: dict = await _read_request_body( request=_http_request ) - logging_payload = ManagementEndpointLoggingPayload( - route=_route, - request_data=_request_body, - response=None, - start_time=start_time, - end_time=end_time, - exception=e, - ) + else: + _route = func.__name__ + _request_body = {} - await open_telemetry_logger.async_management_endpoint_failure_hook( # type: ignore - logging_payload=logging_payload, - parent_otel_span=parent_otel_span, - ) + logging_payload = ManagementEndpointLoggingPayload( + route=_route, + request_data=_request_body, + response=None, + start_time=start_time, + end_time=end_time, + exception=e, + ) + + await open_telemetry_logger.async_management_endpoint_failure_hook( # type: ignore + logging_payload=logging_payload, + parent_otel_span=parent_otel_span, + ) raise e diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 759534a32a..4af7caead0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -550,6 +550,7 @@ from fastapi import ( status, ) from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.openapi.docs import get_swagger_ui_html from fastapi.openapi.utils import get_openapi @@ -1209,15 +1210,69 @@ async def openai_exception_handler(request: Request, exc: ProxyException): # NOTE: DO NOT MODIFY THIS, its crucial to map to Openai exceptions headers = exc.headers error_dict = exc.to_dict() + status_code = int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR + _close_dangling_otel_server_span(request, status_code) return JSONResponse( - status_code=( - int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR - ), + status_code=status_code, content={"error": error_dict}, headers=headers, ) +def _close_dangling_otel_server_span(request: Request, status_code: int) -> None: + parent_otel_span = getattr(request.state, "parent_otel_span", None) + if parent_otel_span is None: + return + if open_telemetry_logger is None: + return + try: + from opentelemetry.trace import Status, StatusCode + + open_telemetry_logger.set_response_status_code_attribute( + parent_otel_span, status_code + ) + parent_otel_span.set_status( + Status(StatusCode.ERROR if status_code >= 400 else StatusCode.OK) + ) + parent_otel_span.end() + except Exception as e: + verbose_proxy_logger.debug( + "Error closing dangling OTEL SERVER span: %s", str(e) + ) + finally: + request.state.parent_otel_span = None + + +@app.exception_handler(RequestValidationError) +async def otel_request_validation_exception_handler( + request: Request, exc: RequestValidationError +): + _close_dangling_otel_server_span(request, 422) + return JSONResponse( + status_code=422, + content={"detail": jsonable_encoder(exc.errors())}, + ) + + +@app.exception_handler(Exception) +async def otel_unhandled_exception_handler(request: Request, exc: Exception): + if isinstance(exc, (ProxyException, HTTPException, RequestValidationError)): + raise exc + verbose_proxy_logger.exception( + "Unhandled exception in request: %s", type(exc).__name__ + ) + _close_dangling_otel_server_span(request, 500) + return JSONResponse( + status_code=500, + content={ + "error": { + "message": "Internal server error", + "type": "internal_server_error", + } + }, + ) + + router = APIRouter() diff --git a/tests/test_litellm/integrations/open_telemetry/__init__.py b/tests/test_litellm/integrations/open_telemetry/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_litellm/integrations/open_telemetry/_helpers.py b/tests/test_litellm/integrations/open_telemetry/_helpers.py new file mode 100644 index 0000000000..69e9daf67c --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/_helpers.py @@ -0,0 +1,109 @@ +""" +Helpers for the LIT-3193 OTEL HTTP-attribute matrix. + +Module split from ``conftest.py`` because pytest auto-discovers fixtures but +forbids ``from .conftest import …`` (no parent package). Fixtures stay in +``conftest.py``; pure helpers (assertions, exception factories, attribute +constants) live here so test modules can ``from ._helpers import …``. +""" + +from typing import Any, Optional + +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.trace import StatusCode + +from litellm.integrations.opentelemetry import ( + HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE, + HTTP_ROUTE_ATTRIBUTE, + LITELLM_PROXY_REQUEST_SPAN_NAME, + URL_PATH_ATTRIBUTE, +) + + +def get_server_span(exporter: InMemorySpanExporter): + """Return the (single) finished SERVER span, or None if it never ended.""" + for s in exporter.get_finished_spans(): + if s.name == LITELLM_PROXY_REQUEST_SPAN_NAME: + return s + return None + + +def assert_server_span_attrs( + exporter: InMemorySpanExporter, + *, + expected_status: int, + expected_url_path: str, + expected_http_route: Optional[str] = None, + where: str = "", +) -> None: + """The four required attributes on the SERVER span must all be set.""" + span = get_server_span(exporter) + assert span is not None, ( + f"{where}: SERVER span never finished — exporter saw " + f"{[s.name for s in exporter.get_finished_spans()]}" + ) + + actual_status = span.attributes.get(HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE) + assert actual_status == expected_status, ( + f"{where}: {HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE} = " + f"{actual_status!r}, expected {expected_status}" + ) + assert isinstance( + actual_status, int + ), f"{where}: status code must be int (semconv), got {type(actual_status)}" + + actual_url = span.attributes.get(URL_PATH_ATTRIBUTE) + assert actual_url == expected_url_path, ( + f"{where}: {URL_PATH_ATTRIBUTE} = {actual_url!r}, " + f"expected {expected_url_path!r}" + ) + + expected_route = expected_http_route or expected_url_path + actual_route = span.attributes.get(HTTP_ROUTE_ATTRIBUTE) + assert actual_route == expected_route, ( + f"{where}: {HTTP_ROUTE_ATTRIBUTE} = {actual_route!r}, " + f"expected {expected_route!r}" + ) + + duration_ns = (span.end_time or 0) - (span.start_time or 0) + assert duration_ns > 0, f"{where}: duration must be > 0, got {duration_ns}ns" + + expected_span_status = StatusCode.ERROR if expected_status >= 400 else StatusCode.OK + actual_span_status = span.status.status_code + assert actual_span_status == expected_span_status, ( + f"{where}: span.status = {actual_span_status!r}, " + f"expected {expected_span_status!r}" + ) + + +# --------------------------------------------------------------------------- +# Synthetic exceptions covering the matrix triggers +# --------------------------------------------------------------------------- +class HttpStatusException(Exception): + """Generic exception with .status_code; mirrors what proxy code reads.""" + + def __init__(self, status_code: int, message: str = "boom"): + super().__init__(message) + self.status_code = status_code + self.code = status_code + + +def make_httpx_status_error(status_code: int, body: str = "upstream error"): + """Real httpx.HTTPStatusError — what providers emit on 4xx/5xx upstream.""" + import httpx + + request = httpx.Request("POST", "https://upstream.example/v1/x") + response = httpx.Response( + status_code=status_code, content=body.encode("utf-8"), request=request + ) + return httpx.HTTPStatusError( + f"HTTP {status_code}", request=request, response=response + ) + + +def make_fastapi_http_exception(status_code: int, detail: Any = "boom"): + from fastapi import HTTPException + + return HTTPException(status_code=status_code, detail=detail) diff --git a/tests/test_litellm/integrations/open_telemetry/conftest.py b/tests/test_litellm/integrations/open_telemetry/conftest.py new file mode 100644 index 0000000000..b29335aedd --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/conftest.py @@ -0,0 +1,93 @@ +""" +Shared fixtures for the LIT-3193 OTEL HTTP-attribute matrix. + +The matrix needs every error response — across unified inference, passthrough, +and admin endpoints — to carry ``http.response.status_code``, ``url.path``, +``http.route``, and a non-zero duration on the SERVER (root) span. These +fixtures hook a real ``OpenTelemetry`` callback into ``litellm.callbacks`` so +the tests drive the actual handler / wrapper code under test, not the OTEL +emitter in isolation. + +See ``LIT-3193_test_matrix.md`` (same directory) for the cell list. +""" + +import os +import sys +from datetime import datetime +from typing import Optional, Tuple +from unittest.mock import MagicMock + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + +sys.path.insert(0, os.path.abspath("../../..")) + +import litellm +from litellm.integrations.opentelemetry import OpenTelemetry + + +# --------------------------------------------------------------------------- +# OTEL + exporter +# --------------------------------------------------------------------------- +@pytest.fixture +def otel_with_exporter() -> Tuple[OpenTelemetry, InMemorySpanExporter]: + """Real OpenTelemetry callback with every span captured in-memory.""" + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + + otel = OpenTelemetry() + otel.tracer = provider.get_tracer("lit-3193-tests") + otel.message_logging = True + return otel, exporter + + +@pytest.fixture +def server_span_factory(otel_with_exporter): + """Factory mirroring user_api_key_auth: SERVER span + url.path + http.route.""" + otel, _exporter = otel_with_exporter + + def _make(url_path: str, http_route: Optional[str] = None): + span = otel.create_litellm_proxy_request_started_span( + start_time=datetime.now(), headers={} + ) + otel.set_proxy_request_route_attributes( + span, url_path=url_path, http_route=http_route or url_path + ) + return span + + return _make + + +@pytest.fixture +def user_api_key_dict_factory(): + """UserAPIKeyAuth-shaped mock; the only attr the failure hooks read is + parent_otel_span (plus team_id/team_alias for stamping).""" + + def _make(parent_span): + d = MagicMock() + d.parent_otel_span = parent_span + d.team_id = "team-lit-3193" + d.team_alias = "lit-3193-team" + d.request_route = None + return d + + return _make + + +@pytest.fixture +def register_otel_callback(otel_with_exporter, monkeypatch): + """Make ProxyLogging.post_call_failure_hook iterate our OTEL instance.""" + otel, _ = otel_with_exporter + saved = list(litellm.callbacks) + monkeypatch.setattr(litellm, "callbacks", [otel]) + yield otel + litellm.callbacks = saved + + +# Helpers (assertions, exception factories) live in ``_helpers.py`` — pytest +# auto-discovers fixtures here but forbids ``from .conftest import …``. diff --git a/tests/test_litellm/integrations/open_telemetry/test_otel_admin_endpoints.py b/tests/test_litellm/integrations/open_telemetry/test_otel_admin_endpoints.py new file mode 100644 index 0000000000..b1a3b834c3 --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/test_otel_admin_endpoints.py @@ -0,0 +1,182 @@ +"""LIT-3193 — admin / management endpoints. Drives the +async_management_endpoint_{success,failure}_hook integration points.""" + +import asyncio +from datetime import datetime + +import pytest + +from litellm.proxy._types import ( + ManagementEndpointLoggingPayload, + UserAPIKeyAuth, +) + +from ._helpers import ( + HttpStatusException, + assert_server_span_attrs, + make_fastapi_http_exception, + make_httpx_status_error, +) + + +def _real_user_api_key_dict(parent_span): + return UserAPIKeyAuth( + api_key="sk-test-admin", + team_id="team-lit-3193", + team_alias="lit-3193-team", + parent_otel_span=parent_span, + ) + + +async def _drive_admin_failure(*, otel, exception, parent_span, route): + payload = ManagementEndpointLoggingPayload( + route=route, + request_data={}, + response=None, + start_time=datetime.now(), + end_time=datetime.now(), + exception=exception, + ) + await otel.async_management_endpoint_failure_hook( + logging_payload=payload, + parent_otel_span=parent_span, + ) + + +async def _drive_admin_success(*, otel, parent_span, route, response): + payload = ManagementEndpointLoggingPayload( + route=route, + request_data={}, + response=response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + await otel.async_management_endpoint_success_hook( + logging_payload=payload, + parent_otel_span=parent_span, + ) + + +KEY_GENERATE_PATH = "/key/generate" + + +@pytest.mark.parametrize( + "exception, expected_status", + [ + (make_fastapi_http_exception(400, "negative max_budget"), 400), + (make_fastapi_http_exception(401, "missing master key"), 401), + (make_fastapi_http_exception(403, "non-admin"), 403), + (make_fastapi_http_exception(422, "validation"), 422), + (HttpStatusException(500, "DB unreachable"), 500), + # Pins .response.status_code fallback through the admin path. + (make_httpx_status_error(500, "upstream blew up"), 500), + ], + ids=["400", "401", "403", "422", "500", "500-httpx"], +) +def test_key_generate_failure_stamps_server_span( + exception, + expected_status, + server_span_factory, + otel_with_exporter, +): + otel, exporter = otel_with_exporter + server_span = server_span_factory(KEY_GENERATE_PATH) + + asyncio.run( + _drive_admin_failure( + otel=otel, + exception=exception, + parent_span=server_span, + route=KEY_GENERATE_PATH, + ) + ) + + assert_server_span_attrs( + exporter, + expected_status=expected_status, + expected_url_path=KEY_GENERATE_PATH, + where=f"key/generate {expected_status}", + ) + + +def test_key_generate_success_stamps_server_span( + server_span_factory, otel_with_exporter +): + otel, exporter = otel_with_exporter + server_span = server_span_factory(KEY_GENERATE_PATH) + + asyncio.run( + _drive_admin_success( + otel=otel, + parent_span=server_span, + route=KEY_GENERATE_PATH, + response={"key": "sk-1", "key_name": "k"}, + ) + ) + + assert_server_span_attrs( + exporter, + expected_status=200, + expected_url_path=KEY_GENERATE_PATH, + where="key/generate 200", + ) + + +SMOKE_ADMIN_ENDPOINTS = [ + "/key/info", + "/key/update", + "/key/delete", + "/team/new", + "/team/member_add", + "/user/new", + "/user/info", + "/model/new", + "/model/delete", + "/customer/new", + "/customer/info", + "/organization/new", + "/organization/member_add", + "/budget/new", + "/budget/info", + "/credentials/new", + "/mcp/server/add", + "/tag/new", +] + + +@pytest.mark.parametrize("path", SMOKE_ADMIN_ENDPOINTS) +@pytest.mark.parametrize( + "exception, expected_status", + [ + (make_fastapi_http_exception(404, "not found"), 404), + (HttpStatusException(500, "DB unreachable"), 500), + ], + ids=["404", "500"], +) +def test_admin_endpoint_failure_stamps_server_span( + path, + exception, + expected_status, + server_span_factory, + otel_with_exporter, +): + """Confirm SERVER-span stamping works for every admin resource family — + same wrapper, just different routes.""" + otel, exporter = otel_with_exporter + server_span = server_span_factory(path) + + asyncio.run( + _drive_admin_failure( + otel=otel, + exception=exception, + parent_span=server_span, + route=path, + ) + ) + + assert_server_span_attrs( + exporter, + expected_status=expected_status, + expected_url_path=path, + where=f"{path} {expected_status}", + ) diff --git a/tests/test_litellm/integrations/open_telemetry/test_otel_exception_handler.py b/tests/test_litellm/integrations/open_telemetry/test_otel_exception_handler.py new file mode 100644 index 0000000000..56059c260c --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/test_otel_exception_handler.py @@ -0,0 +1,136 @@ +"""LIT-3193 — exception-handler path. Closes SERVER spans for requests +that fail after auth but before the route handler runs (e.g. /model/new +TypeError or RequestValidationError).""" + +import asyncio +import types + +import pytest +from fastapi import HTTPException +from fastapi.exceptions import RequestValidationError + +import litellm.proxy.proxy_server as proxy_server_module +from litellm.proxy._types import ProxyException +from litellm.proxy.proxy_server import ( + _close_dangling_otel_server_span, + openai_exception_handler, + otel_request_validation_exception_handler, + otel_unhandled_exception_handler, +) + +from ._helpers import assert_server_span_attrs + + +def _fake_request(parent_otel_span=None): + state = types.SimpleNamespace() + if parent_otel_span is not None: + state.parent_otel_span = parent_otel_span + return types.SimpleNamespace(state=state) + + +@pytest.fixture +def wired_otel(otel_with_exporter, monkeypatch): + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server_module, "open_telemetry_logger", otel) + return exporter + + +@pytest.mark.parametrize("status,path", [(500, "/model/new"), (422, "/key/generate")]) +def test_close_dangling_span_stamps_status( + wired_otel, server_span_factory, status, path +): + request = _fake_request(parent_otel_span=server_span_factory(path)) + _close_dangling_otel_server_span(request, status) + assert_server_span_attrs( + wired_otel, + expected_status=status, + expected_url_path=path, + where=f"{path} {status}", + ) + assert request.state.parent_otel_span is None + + +def test_close_dangling_span_noop_when_no_span(wired_otel): + _close_dangling_otel_server_span(_fake_request(), 500) + assert wired_otel.get_finished_spans() == () + + +def test_close_dangling_span_noop_when_otel_absent(server_span_factory, monkeypatch): + monkeypatch.setattr(proxy_server_module, "open_telemetry_logger", None) + request = _fake_request(parent_otel_span=server_span_factory("/key/generate")) + _close_dangling_otel_server_span(request, 500) + + +@pytest.mark.parametrize( + "handler,exc,status,path", + [ + ( + otel_request_validation_exception_handler, + RequestValidationError(errors=[]), + 422, + "/key/generate", + ), + ( + otel_unhandled_exception_handler, + TypeError("Deployment.__init__() missing required positional arg"), + 500, + "/model/new", + ), + ], +) +def test_exception_handler_closes_span( + wired_otel, server_span_factory, handler, exc, status, path +): + request = _fake_request(parent_otel_span=server_span_factory(path)) + response = asyncio.run(handler(request, exc)) + assert response.status_code == status + assert_server_span_attrs( + wired_otel, + expected_status=status, + expected_url_path=path, + where=f"{handler.__name__} ({type(exc).__name__})", + ) + + +def test_unhandled_handler_reraises_known_exceptions(wired_otel, server_span_factory): + """ProxyException / HTTPException / RequestValidationError have dedicated handlers.""" + request = _fake_request(parent_otel_span=server_span_factory("/key/generate")) + with pytest.raises(HTTPException): + asyncio.run( + otel_unhandled_exception_handler( + request, HTTPException(status_code=403, detail="forbidden") + ) + ) + + +# Covers ProxyException raised after auth stashed the span (e.g., invalid-JSON +# body via _read_request_body) — handler must close the dangling SERVER span. +@pytest.mark.parametrize( + "code,path", + [ + (400, "/v1/chat/completions"), + (400, "/v1/messages"), + (400, "/v1/responses"), + (429, "/v1/chat/completions"), + (503, "/v1/chat/completions"), + ], +) +def test_openai_exception_handler_closes_span( + wired_otel, server_span_factory, code, path +): + request = _fake_request(parent_otel_span=server_span_factory(path)) + exc = ProxyException( + message="boom", + type="invalid_request_error", + param="request_body", + code=code, + ) + response = asyncio.run(openai_exception_handler(request, exc)) + assert response.status_code == code + assert_server_span_attrs( + wired_otel, + expected_status=code, + expected_url_path=path, + where=f"openai_exception_handler ({path} code={code})", + ) + assert request.state.parent_otel_span is None diff --git a/tests/test_litellm/integrations/open_telemetry/test_otel_passthrough_endpoints.py b/tests/test_litellm/integrations/open_telemetry/test_otel_passthrough_endpoints.py new file mode 100644 index 0000000000..4b24b6c487 --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/test_otel_passthrough_endpoints.py @@ -0,0 +1,136 @@ +"""LIT-3193 — passthrough endpoints. Drives proxy_logging.post_call_failure_hook +(the integration point pass_through_endpoint reaches on upstream >=300).""" + +import asyncio + +import pytest +from fastapi import HTTPException + +from litellm.caching.dual_cache import DualCache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import ProxyLogging + +from ._helpers import ( + assert_server_span_attrs, + make_fastapi_http_exception, + make_httpx_status_error, +) + + +def _real_user_api_key_dict(parent_span): + return UserAPIKeyAuth( + api_key="sk-test", + team_id="team-lit-3193", + team_alias="lit-3193-team", + parent_otel_span=parent_span, + ) + + +def _proxy_logging(): + return ProxyLogging(user_api_key_cache=UserApiKeyCache(DualCache())) + + +def _drive_passthrough_failure(*, exception, user_api_key_dict): + asyncio.run( + _proxy_logging().post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=exception, + request_data={}, + ) + ) + + +VERTEX_PATH = "/vertex_ai/v1/projects/p/locations/us-central1/publishers/google/models/gemini-1.5-pro:generateContent" + + +@pytest.mark.parametrize( + "exception, expected_status", + [ + (make_fastapi_http_exception(401, "no proxy key"), 401), + (make_fastapi_http_exception(400, "bad request"), 400), + (make_fastapi_http_exception(403, "upstream forbidden"), 403), + (make_fastapi_http_exception(404, "upstream not found"), 404), + (make_fastapi_http_exception(429, "upstream rate limit"), 429), + (make_httpx_status_error(500, "upstream blew up"), 500), + (make_httpx_status_error(502, "bad gateway"), 502), + (make_httpx_status_error(503, "service unavailable"), 503), + (make_fastapi_http_exception(502, "wrapped 502"), 502), + ], + ids=[ + "401-litellm-auth", + "400-upstream", + "403-upstream", + "404-upstream", + "429-upstream", + "500-upstream-httpx", + "502-upstream-httpx", + "503-upstream-httpx", + "502-wrapped", + ], +) +def test_vertex_passthrough_failure_stamps_server_span( + exception, + expected_status, + server_span_factory, + otel_with_exporter, + register_otel_callback, +): + _otel, exporter = otel_with_exporter + server_span = server_span_factory( + VERTEX_PATH, http_route="/vertex_ai/{endpoint:path}" + ) + uakd = _real_user_api_key_dict(server_span) + + _drive_passthrough_failure(exception=exception, user_api_key_dict=uakd) + + assert_server_span_attrs( + exporter, + expected_status=expected_status, + expected_url_path=VERTEX_PATH, + expected_http_route="/vertex_ai/{endpoint:path}", + where=f"vertex passthrough {expected_status}", + ) + + +SMOKE_PASSTHROUGHS = [ + ("/bedrock/model/anthropic.claude-v2/invoke", "/bedrock/{endpoint:path}"), + ("/anthropic/v1/messages", "/anthropic/{endpoint:path}"), + ("/openai/v1/chat/completions", "/openai/{endpoint:path}"), + ("/gemini/v1beta/models/gemini-pro:generateContent", "/gemini/{endpoint:path}"), + ("/cohere/v1/chat", "/cohere/{endpoint:path}"), + ("/azure/openai/deployments/gpt4/chat/completions", "/azure/{endpoint:path}"), +] + + +@pytest.mark.parametrize("path,http_route", SMOKE_PASSTHROUGHS) +@pytest.mark.parametrize( + "exception, expected_status", + [ + (make_fastapi_http_exception(400, "upstream bad request"), 400), + (make_httpx_status_error(502, "upstream"), 502), + ], + ids=["400", "502"], +) +def test_passthrough_failure_stamps_server_span( + path, + http_route, + exception, + expected_status, + server_span_factory, + otel_with_exporter, + register_otel_callback, +): + _otel, exporter = otel_with_exporter + server_span = server_span_factory(path, http_route=http_route) + uakd = _real_user_api_key_dict(server_span) + + _drive_passthrough_failure(exception=exception, user_api_key_dict=uakd) + + assert_server_span_attrs( + exporter, + expected_status=expected_status, + expected_url_path=path, + expected_http_route=http_route, + where=f"{path} {expected_status}", + ) diff --git a/tests/test_litellm/integrations/open_telemetry/test_otel_unified_endpoints.py b/tests/test_litellm/integrations/open_telemetry/test_otel_unified_endpoints.py new file mode 100644 index 0000000000..0279c22154 --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/test_otel_unified_endpoints.py @@ -0,0 +1,223 @@ +"""LIT-3193 — unified inference endpoints. Drives _handle_llm_api_exception +to assert SERVER-span attrs (status, url.path, http.route, duration).""" + +import asyncio + +import pytest +from fastapi import HTTPException + +from opentelemetry.trace import Status, StatusCode + +import litellm +from litellm.caching.dual_cache import DualCache +from litellm.proxy._types import ProxyException, UserAPIKeyAuth +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import ProxyLogging + +from ._helpers import ( + HttpStatusException, + assert_server_span_attrs, + make_fastapi_http_exception, + make_httpx_status_error, +) + + +def _real_user_api_key_dict(parent_span): + return UserAPIKeyAuth( + api_key="sk-test", + team_id="team-lit-3193", + team_alias="lit-3193-team", + parent_otel_span=parent_span, + ) + + +def _proxy_logging(): + return ProxyLogging(user_api_key_cache=UserApiKeyCache(DualCache())) + + +def _drive_unified_failure( + *, + exception, + server_span, + user_api_key_dict, +): + proc = ProxyBaseLLMRequestProcessing(data={}) + try: + asyncio.run( + proc._handle_llm_api_exception( + e=exception, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=_proxy_logging(), + ) + ) + except (ProxyException, HTTPException): + pass + + +CHAT_PATH = "/v1/chat/completions" + + +@pytest.mark.parametrize( + "exception, expected_status", + [ + (make_fastapi_http_exception(400, "bad request"), 400), + (make_fastapi_http_exception(401, "no key"), 401), + (make_fastapi_http_exception(403, "no model access"), 403), + (make_fastapi_http_exception(404, "model not in router"), 404), + (make_fastapi_http_exception(422, "validation"), 422), + (make_fastapi_http_exception(429, "rate limit"), 429), + (HttpStatusException(500, "uncaught"), 500), + (make_httpx_status_error(502, "upstream blew up"), 502), + (make_httpx_status_error(503, "upstream down"), 503), + (make_httpx_status_error(504, "upstream timeout"), 504), + ], + ids=[ + "400-bad-request", + "401-no-key", + "403-no-model-access", + "404-model-not-found", + "422-validation", + "429-rate-limit", + "500-uncaught", + "502-upstream", + "503-upstream", + "504-upstream-timeout", + ], +) +def test_chat_completions_failure_stamps_server_span( + exception, + expected_status, + server_span_factory, + user_api_key_dict_factory, + otel_with_exporter, + register_otel_callback, +): + _otel, exporter = otel_with_exporter + server_span = server_span_factory(CHAT_PATH) + uakd = _real_user_api_key_dict(server_span) + + _drive_unified_failure( + exception=exception, server_span=server_span, user_api_key_dict=uakd + ) + + assert_server_span_attrs( + exporter, + expected_status=expected_status, + expected_url_path=CHAT_PATH, + where=f"chat/completions {expected_status}", + ) + + +def test_chat_completions_success_path_stamps_200( + otel_with_exporter, server_span_factory +): + otel, exporter = otel_with_exporter + server_span = server_span_factory(CHAT_PATH) + _real_user_api_key_dict(server_span) + + otel.set_response_status_code_attribute(server_span, 200) + otel.set_preprocessing_duration_attribute(server_span, {}) + server_span.set_status(Status(StatusCode.OK)) + server_span.end() + + assert_server_span_attrs( + exporter, + expected_status=200, + expected_url_path=CHAT_PATH, + where="chat/completions 200", + ) + + +# /v1/responses ends the proxy span before async_post_call_success_hook fires, +# so the 200 stamp must happen at span close (here), not in the hook. +@pytest.mark.parametrize( + "path", ["/v1/chat/completions", "/v1/messages", "/v1/responses"] +) +def test_end_proxy_span_from_kwargs_stamps_200( + path, otel_with_exporter, server_span_factory +): + from datetime import datetime + + otel, exporter = otel_with_exporter + server_span = server_span_factory(path) + kwargs = {"litellm_params": {"metadata": {"litellm_parent_otel_span": server_span}}} + otel._end_proxy_span_from_kwargs(kwargs, datetime.now()) + + assert_server_span_attrs( + exporter, + expected_status=200, + expected_url_path=path, + where=f"{path} _end_proxy_span_from_kwargs", + ) + + +# Bare TypeError has no .code/.status_code, so error_information.error_code is +# empty and _record_exception_on_span skips the stamp — must default to 500. +def test_async_post_call_failure_hook_defaults_to_500( + otel_with_exporter, server_span_factory +): + otel, exporter = otel_with_exporter + server_span = server_span_factory("/v1/responses") + uakd = _real_user_api_key_dict(server_span) + + asyncio.run( + otel.async_post_call_failure_hook( + request_data={}, + original_exception=TypeError("missing required argument"), + user_api_key_dict=uakd, + ) + ) + + assert_server_span_attrs( + exporter, + expected_status=500, + expected_url_path="/v1/responses", + where="async_post_call_failure_hook (TypeError) defaults to 500", + ) + + +SMOKE_ENDPOINTS = [ + "/v1/embeddings", + "/v1/completions", + "/v1/images/generations", + "/v1/audio/speech", + "/v1/audio/transcriptions", + "/v1/moderations", + "/v1/rerank", + "/v1/responses", + "/v1/messages", +] + + +@pytest.mark.parametrize("path", SMOKE_ENDPOINTS) +@pytest.mark.parametrize( + "exception, expected_status", + [ + (make_fastapi_http_exception(401, "no key"), 401), + (make_httpx_status_error(502, "upstream"), 502), + ], + ids=["401", "502"], +) +def test_unified_endpoint_failure_stamps_server_span( + path, + exception, + expected_status, + server_span_factory, + otel_with_exporter, + register_otel_callback, +): + _otel, exporter = otel_with_exporter + server_span = server_span_factory(path) + uakd = _real_user_api_key_dict(server_span) + + _drive_unified_failure( + exception=exception, server_span=server_span, user_api_key_dict=uakd + ) + + assert_server_span_attrs( + exporter, + expected_status=expected_status, + expected_url_path=path, + where=f"{path} {expected_status}", + )