diff --git a/litellm/llms/base_llm/managed_resources/__init__.py b/litellm/llms/base_llm/managed_resources/__init__.py index 5eb9b46f89..a5543e631c 100644 --- a/litellm/llms/base_llm/managed_resources/__init__.py +++ b/litellm/llms/base_llm/managed_resources/__init__.py @@ -24,10 +24,12 @@ from .utils import ( generate_unified_id_string, is_base64_encoded_unified_id, parse_unified_id, + resolve_passthrough_managed_id_provider, ) __all__ = [ "BaseManagedResource", + "resolve_passthrough_managed_id_provider", "is_base64_encoded_unified_id", "extract_target_model_names_from_unified_id", "extract_resource_type_from_unified_id", diff --git a/litellm/llms/base_llm/managed_resources/utils.py b/litellm/llms/base_llm/managed_resources/utils.py index 6e30b6cb25..e9a6aef689 100644 --- a/litellm/llms/base_llm/managed_resources/utils.py +++ b/litellm/llms/base_llm/managed_resources/utils.py @@ -7,7 +7,40 @@ different managed resource types (files, vector stores, etc.). import base64 import re -from typing import List, Optional, Union, Literal +from typing import Any, List, Literal, Optional, Union + +PASSTHROUGH_MANAGED_ID_AZURE_PROVIDERS = ("azure", "azure_ai") + + +def resolve_passthrough_managed_id_provider( + custom_llm_provider: Any, +) -> Optional[str]: + """Map a pass-through ``custom_llm_provider`` to the provider scope that + namespaces passthrough managed object IDs, or ``None`` when the route is not + an OpenAI/Azure pass-through and managed IDs must not apply. + + Scoping is keyed on the explicit provider that the pass-through route + forwards (``openai``, ``azure``, ``azure_ai``), not on the upstream URL, so + a third-party OpenAI-compatible endpoint never triggers managed-ID minting. + + ``azure`` and ``azure_ai`` deliberately collapse to one ``"azure"`` scope: + they expose the same Azure OpenAI files/batches surface, so an ID minted + while routing as one must still resolve while routing as the other. + Splitting them would make a managed ID minted on ``azure`` fail to resolve + when replayed on ``azure_ai`` and vice versa. + """ + provider = str( + getattr(custom_llm_provider, "value", custom_llm_provider) or "" + ).lower() + if not provider: + return None + if provider in PASSTHROUGH_MANAGED_ID_AZURE_PROVIDERS or provider.endswith( + (".azure", ".azure_ai") + ): + return "azure" + if provider == "openai" or provider.endswith(".openai"): + return "openai" + return None def is_base64_encoded_unified_id( diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 7ca28a5d4a..e94f56302a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -2091,6 +2091,11 @@ class BaseOpenAIPassThroughHandler: api_key=api_key, request=request, extra_headers=extra_headers ), is_streaming_request=is_streaming_request, # type: ignore + custom_llm_provider=( + custom_llm_provider.value + if hasattr(custom_llm_provider, "value") + else str(custom_llm_provider) if custom_llm_provider else None + ), ) # dynamically construct pass-through endpoint based on incoming path received_value = await endpoint_func( request, diff --git a/litellm/proxy/pass_through_endpoints/managed_id_codec.py b/litellm/proxy/pass_through_endpoints/managed_id_codec.py new file mode 100644 index 0000000000..f0c24bbaf3 --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/managed_id_codec.py @@ -0,0 +1,97 @@ +""" +Codec for LiteLLM passthrough-managed object IDs. + +Plaintext format (before urlsafe-base64 encoding): + litellm_proxy:passthrough;provider:{p};unified_id,{u};raw_id,{r} + +Uses the same base64.urlsafe_b64encode / padding-restore convention as +``_is_base64_encoded_unified_file_id`` in +``openai_files_endpoints/common_utils.py``. + +The ``passthrough;`` discriminator distinguishes these rows from +unified-endpoint rows that share the same LiteLLM_ManagedFileTable / +LiteLLM_ManagedObjectTable. ``_resolve_one`` in the rewriter module rejects +any row whose decoded plaintext lacks this discriminator, making cross-system +replay safe. +""" + +from __future__ import annotations + +import base64 +import uuid as _uuid_mod +from dataclasses import dataclass +from typing import Optional + +from litellm.types.utils import SpecialEnums + +_PREFIX = SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value # "litellm_proxy" +_DISCRIMINATOR = "passthrough" + + +@dataclass(frozen=True) +class ManagedIdPayload: + """Decoded contents of a passthrough managed ID.""" + + provider: str + unified_uuid: str + raw_provider_id: str + + +def encode(provider: str, unified_uuid: str, raw_provider_id: str) -> str: + """Return a urlsafe-base64 managed ID string (trailing ``=`` stripped).""" + plaintext = SpecialEnums.LITELLM_PASSTHROUGH_MANAGED_ID_COMPLETE_STR.value.format( + provider, unified_uuid, raw_provider_id + ) + return base64.urlsafe_b64encode(plaintext.encode()).decode().rstrip("=") + + +def decode(managed_id: str) -> Optional[ManagedIdPayload]: + """ + Decode *managed_id*. + + Returns ``None`` for anything that is not a passthrough managed ID — raw + OpenAI IDs, unified-endpoint IDs, garbage, wrong types. Never raises. + """ + if not isinstance(managed_id, str): + return None + # Restore stripped padding before decoding + padded = managed_id + "=" * (-len(managed_id) % 4) + try: + plaintext = base64.urlsafe_b64decode(padded).decode() + except Exception: + return None + + # Must start with "litellm_proxy:passthrough;" + expected_head = f"{_PREFIX}:{_DISCRIMINATOR};" + if not plaintext.startswith(expected_head): + return None + + rest = plaintext[len(expected_head) :] + try: + # Split only on first two ';' so a raw_id containing ';' cannot + # break parsing (OpenAI IDs don't use ';', but defensive). + provider_part, rest2 = rest.split(";", 1) + unified_part, raw_id_part = rest2.split(";", 1) + if not ( + provider_part.startswith("provider:") + and unified_part.startswith("unified_id,") + and raw_id_part.startswith("raw_id,") + ): + return None + return ManagedIdPayload( + provider=provider_part[len("provider:") :], + unified_uuid=unified_part[len("unified_id,") :], + raw_provider_id=raw_id_part[len("raw_id,") :], + ) + except Exception: + return None + + +def is_managed(value: str) -> bool: + """Return ``True`` iff *value* decodes to a passthrough managed ID.""" + return decode(value) is not None + + +def new_managed_id(provider: str, raw_provider_id: str) -> str: + """Mint a fresh managed ID for a given raw provider ID.""" + return encode(provider, str(_uuid_mod.uuid4()), raw_provider_id) diff --git a/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py b/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py new file mode 100644 index 0000000000..a267c97c0e --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py @@ -0,0 +1,1234 @@ +""" +Rewrite passthrough-managed IDs in pass-through endpoint requests and responses. + +OUTPUT (response) path +---------------------- +``rewrite_response_ids()`` is called after the upstream response is received. +It looks up the (provider, method, path) combination in ``BUILTIN_OUTPUT_ID_FIELD_MAP``, +mints a managed ID for each listed field whose raw provider value is present, +stores / reuses a DB row (dedup), and swaps the value in the body before the +response is returned to the client. + +INPUT (request) path +-------------------- +``rewrite_path_ids()``, ``rewrite_query_ids()``, and ``rewrite_body_ids()`` +are called just before the request is forwarded upstream. Each one walks its +respective location (URL path, query params, JSON body) and calls +``_resolve_one()`` for every string that looks like a passthrough managed ID +(decode-first detection). ``_resolve_one()`` enforces: + + 1. Cross-route check: the provider embedded in the ID must match the current + route's provider, else HTTPException(404). + 2. DB existence check: unknown / forged IDs raise HTTPException(404); the + raw string is NEVER forwarded to upstream. + 3. Access check: ``can_access_resource()`` raises HTTPException(403) on + mismatch. + +When a value does not decode as a passthrough managed ID it is passed through +untouched (deliberate opt-out for raw OpenAI IDs). +""" + +from __future__ import annotations + +import json +import re +from typing import Any, Dict, FrozenSet, List, Optional, Tuple +from urllib.parse import quote, unquote + +from fastapi import HTTPException + +from litellm._logging import verbose_proxy_logger +from litellm.llms.base_llm.managed_resources.isolation import ( + build_owner_filter, + can_access_resource, +) +from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.llms.openai import OpenAIFileObject + +from .managed_id_codec import ManagedIdPayload, decode, is_managed, new_managed_id + +# --------------------------------------------------------------------------- +# Field map +# --------------------------------------------------------------------------- + +_FieldSpec = Tuple[str, str] # (field_name, expected_raw_id_prefix) +_MapKey = Tuple[str, str, str] # (provider, HTTP_METHOD, canonical_path) + +# ``canonical_path`` uses ``/v1/...`` form without any ``/openai/`` prefix. +# Both ``/openai/...`` and ``/openai_passthrough/...`` are normalised by +# ``_canonical_path()`` before the lookup so only one set of entries is needed. +BUILTIN_OUTPUT_ID_FIELD_MAP: Dict[_MapKey, List[_FieldSpec]] = { + # ------------------------------------------------------------------ files + ("openai", "POST", "/v1/files"): [ + ("id", "file-"), + ], + ("openai", "GET", "/v1/files/{file_id}"): [ + ("id", "file-"), + ], + ("openai", "DELETE", "/v1/files/{file_id}"): [ + ("id", "file-"), + ], + # ----------------------------------------------------------------- batches + ("openai", "POST", "/v1/batches"): [ + ("id", "batch_"), + ("input_file_id", "file-"), + ("output_file_id", "file-"), + ("error_file_id", "file-"), + ], + ("openai", "GET", "/v1/batches/{batch_id}"): [ + ("id", "batch_"), + ("input_file_id", "file-"), + ("output_file_id", "file-"), + ("error_file_id", "file-"), + ], + ("openai", "POST", "/v1/batches/{batch_id}/cancel"): [ + ("id", "batch_"), + ("input_file_id", "file-"), + ("output_file_id", "file-"), + ("error_file_id", "file-"), + ], + # --------------------------------------------------------------- responses + ("openai", "POST", "/v1/responses"): [ + ("id", "resp_"), + ], + ("openai", "GET", "/v1/responses/{response_id}"): [ + ("id", "resp_"), + ], + ("openai", "DELETE", "/v1/responses/{response_id}"): [ + ("id", "resp_"), + ], + # ================================================================ azure + # Azure OpenAI exposes the same files/batches surface as OpenAI. + # IDs are scoped to "azure" so they are never confused with "openai" ones. + # ------------------------------------------------------------------ files + ("azure", "POST", "/v1/files"): [ + ("id", "file-"), + ], + ("azure", "GET", "/v1/files/{file_id}"): [ + ("id", "file-"), + ], + ("azure", "DELETE", "/v1/files/{file_id}"): [ + ("id", "file-"), + ], + # ----------------------------------------------------------------- batches + ("azure", "POST", "/v1/batches"): [ + ("id", "batch_"), + ("input_file_id", "file-"), + ("output_file_id", "file-"), + ("error_file_id", "file-"), + ], + ("azure", "GET", "/v1/batches/{batch_id}"): [ + ("id", "batch_"), + ("input_file_id", "file-"), + ("output_file_id", "file-"), + ("error_file_id", "file-"), + ], + ("azure", "POST", "/v1/batches/{batch_id}/cancel"): [ + ("id", "batch_"), + ("input_file_id", "file-"), + ("output_file_id", "file-"), + ("error_file_id", "file-"), + ], + # --------------------------------------------------------------- responses + ("azure", "POST", "/v1/responses"): [ + ("id", "resp_"), + ], + ("azure", "GET", "/v1/responses/{response_id}"): [ + ("id", "resp_"), + ], + ("azure", "DELETE", "/v1/responses/{response_id}"): [ + ("id", "resp_"), + ], +} + +# Prefixes that live in the *file* table rather than the object table. +_FILE_PREFIXES: FrozenSet[str] = frozenset({"file-"}) + +# Raw provider-ID prefixes that live in the object table (batches, responses). +_OBJECT_PREFIXES: FrozenSet[str] = frozenset({"batch_", "resp_"}) + +# Guards request-body rewriting against stack exhaustion from adversarially +# deep payloads. Real OpenAI files/batches bodies nest only a few levels. +_MAX_BODY_REWRITE_DEPTH = 64 + +# Caps the distinct raw-provider-id guard lookups issued per request. A raw +# file-id guard is an unindexed array-containment scan over +# LiteLLM_ManagedFileTable (flat_model_file_ids has no index), so a body packed +# with id-shaped strings could otherwise amplify one request into thousands of +# full-table scans. Legitimate callers reference managed IDs (resolved via an +# indexed lookup, never the guard), so guarding more raw ids than this only +# happens under abuse; the request is rejected rather than skipping the guard. +_MAX_RAW_ID_GUARD_LOOKUPS = 100 + + +class _RawIdGuardBudget: + """Per-request de-dupe + cap for raw-provider-id guard DB lookups.""" + + __slots__ = ("_remaining", "_seen") + + def __init__(self, limit: int = _MAX_RAW_ID_GUARD_LOOKUPS) -> None: + self._remaining = limit + self._seen: set = set() + + def reserve(self, raw_id: str) -> bool: + """Return True when a guard lookup for *raw_id* should run. Returns + False for a raw id already checked this request (de-dupe). Raises + ``HTTPException(400)`` once the per-request lookup budget is exhausted.""" + if raw_id in self._seen: + return False + if self._remaining <= 0: + raise HTTPException( + status_code=400, + detail="Too many resource identifiers in request.", + ) + self._remaining -= 1 + self._seen.add(raw_id) + return True + + +# --------------------------------------------------------------------------- +# List routes — GET requests that return a paginated {object:"list", data:[…]} +# These are intercepted and served entirely from the DB rather than forwarded +# to the upstream provider, so each caller only sees IDs they own. +# --------------------------------------------------------------------------- + +# Maps (provider, canonical_path) -> "files" | "batches" +_LIST_ROUTE_TABLE: Dict[Tuple[str, str], str] = { + ("openai", "/v1/files"): "files", + ("openai", "/v1/batches"): "batches", + ("azure", "/v1/files"): "files", + ("azure", "/v1/batches"): "batches", +} + + +# Sentinel model_id written to model_mappings for passthrough-created rows. +# Prevents the unified-endpoint deployment-resolution path from ever finding a +# real deployment, so a passthrough ID replayed on a unified endpoint fails +# cleanly (no silent raw-ID leak). +def _passthrough_sentinel_model_id(provider: str) -> str: + return f"_passthrough_{provider}" + + +# Key under which the provider marker is stored in a file row's model_mappings. +# Its value lands in flat_model_file_ids (built from model_mappings.values()), +# giving the file table a DB-queryable provider scope it otherwise lacks. +_PASSTHROUGH_PROVIDER_MARKER_KEY = "_passthrough_provider_marker" + + +def _passthrough_provider_marker(provider: str) -> str: + return f"_passthrough_provider:{provider}" + + +def _managed_id_matches_provider(unified_id: str, provider: str) -> bool: + payload = decode(unified_id) + return payload is not None and payload.provider == provider + + +# Strip /openai or /openai_passthrough prefix to produce canonical /v1/... path. +# Strips provider-specific passthrough prefixes before the /v1/... path: +# /openai_passthrough/v1/files -> /v1/files +# /openai/v1/files -> /v1/files +# /azure/openai/files -> /files (_canonical_path then prepends /v1/) +# /azure_ai/openai/files -> /files +_PASSTHROUGH_PREFIX_RE = re.compile( + r"^/(?:azure(?:_ai)?/)?openai(?:_passthrough)?(?=/|$)" +) + + +def _canonical_path(route: str) -> str: + """ + Normalise a passthrough route to a bare /v1/... path for map lookup. + + Examples: + /openai_passthrough/v1/files -> /v1/files + /openai/v1/files -> /v1/files + /azure/openai/files -> /v1/files (Azure omits /v1/) + /azure/openai/batches/batch_x -> /v1/batches/batch_x + """ + stripped = _PASSTHROUGH_PREFIX_RE.sub("", route) or "/" + # Azure API paths don't include /v1/ — add it so they match the map keys. + if not stripped.startswith("/v1/") and stripped != "/": + stripped = "/v1" + stripped + return stripped + + +# --------------------------------------------------------------------------- +# Shared resolver — used by all INPUT path extractors +# --------------------------------------------------------------------------- + + +async def _resolve_one( + managed_id: str, + provider: str, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: Any, + managed_files_hook: Any, +) -> str: + """ + Resolve a single value that may be a passthrough managed ID. + + Returns the raw provider ID on success. + Returns *managed_id* unchanged when it is NOT a managed ID so callers + need not pre-filter. + Raises ``HTTPException(403)`` on access denial. + Raises ``HTTPException(404)`` on unknown / forged managed IDs — never + forwarded upstream as a literal string. + """ + payload: Optional[ManagedIdPayload] = decode(managed_id) + if payload is None: + return managed_id # not a passthrough managed ID; pass through + verbose_proxy_logger.debug( + "managed_id_rewriter: resolving managed id provider=%s raw_prefix=%s", + provider, + ( + payload.raw_provider_id.split("_", 1)[0] + if "_" in payload.raw_provider_id + else payload.raw_provider_id.split("-", 1)[0] + ), + ) + + # 1. Cross-route (cross-provider) check + if payload.provider != provider: + raise HTTPException( + status_code=404, + detail=( + f"Managed ID was minted for provider '{payload.provider}', " + f"not '{provider}'." + ), + ) + + row_created_by: Optional[str] = None + row_team_id: Optional[str] = None + found = False + + raw_id = payload.raw_provider_id + + # 2. DB lookup — pick table based on raw ID prefix + if any(raw_id.startswith(p) for p in _FILE_PREFIXES): + # File table — use hook's internal cache for speed when available + if managed_files_hook is not None: + try: + file_row = await managed_files_hook.get_unified_file_id( + managed_id, + litellm_parent_otel_span=None, + ) + if file_row is not None: + row_created_by = file_row.created_by + row_team_id = file_row.team_id + found = True + except Exception: + verbose_proxy_logger.debug( + "managed_id_rewriter._resolve_one: file hook lookup failed", + exc_info=True, + ) + if not found and prisma_client is not None: + try: + db_row = await prisma_client.db.litellm_managedfiletable.find_first( + where={"unified_file_id": managed_id} + ) + if db_row is not None: + row_created_by = db_row.created_by + row_team_id = db_row.team_id + found = True + except Exception: + verbose_proxy_logger.debug( + "managed_id_rewriter._resolve_one: file DB lookup failed", + exc_info=True, + ) + else: + # Object table (batches, responses) + if prisma_client is not None: + try: + obj_row = await prisma_client.db.litellm_managedobjecttable.find_first( + where={"unified_object_id": managed_id} + ) + if obj_row is not None: + row_created_by = obj_row.created_by + row_team_id = obj_row.team_id + found = True + except Exception: + verbose_proxy_logger.debug( + "managed_id_rewriter._resolve_one: object DB lookup failed", + exc_info=True, + ) + + # 3. Hard 404 for unknown / forged IDs — NEVER forward to upstream + if not found: + raise HTTPException( + status_code=404, + detail="Managed resource not found.", + ) + + # 4. Access check + if not can_access_resource(user_api_key_dict, row_created_by, row_team_id): + raise HTTPException( + status_code=403, + detail="Access denied to managed resource.", + ) + + return payload.raw_provider_id + + +async def _guard_raw_provider_id( + raw_id: str, + provider: str, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: Any, + budget: Optional[_RawIdGuardBudget] = None, +) -> None: + """Deny a raw provider ID that maps to a managed resource the caller does + not own, before it is forwarded upstream. + + Clients only ever receive managed IDs (response bodies are rewritten), so a + raw provider ID for another tenant's managed resource can only have been + recovered by decoding that tenant's managed ID. Raw IDs are otherwise + forwarded untouched (deliberate opt-out), which on a retrieve / cancel / + delete would execute upstream before the response-side ownership check ever + runs. Resolving the access check here, on input, keeps the raw fallback + from becoming a cross-tenant bypass. Genuinely unmanaged raw IDs (no DB + row) are left untouched; ``HTTPException(404)`` mirrors the managed-ID + resolver so callers cannot probe which raw IDs exist. + """ + if prisma_client is None: + return + + if any(raw_id.startswith(p) for p in _FILE_PREFIXES): + if budget is not None and not budget.reserve(raw_id): + return + # File rows have no provider column, so fetch every row holding this raw + # id and scope to the current provider in the application layer (same as + # _mint_or_reuse_file's dedup). + try: + candidates = await prisma_client.db.litellm_managedfiletable.find_many( + where={"flat_model_file_ids": {"has": raw_id}}, + ) + except Exception: + verbose_proxy_logger.debug( + "managed_id_rewriter: raw file-id guard lookup failed", exc_info=True + ) + return + provider_rows = [ + row + for row in (candidates or []) + if _managed_id_matches_provider(row.unified_file_id, provider) + ] + if provider_rows and not any( + can_access_resource(user_api_key_dict, row.created_by, row.team_id) + for row in provider_rows + ): + raise HTTPException(status_code=404, detail="Managed resource not found.") + return + + if any(raw_id.startswith(p) for p in _OBJECT_PREFIXES): + if budget is not None and not budget.reserve(raw_id): + return + # Object rows store model_object_id as "passthrough:{provider}:{raw}", so + # the lookup is exact and already provider-scoped. + try: + existing = await prisma_client.db.litellm_managedobjecttable.find_first( + where={"model_object_id": f"passthrough:{provider}:{raw_id}"} + ) + except Exception: + verbose_proxy_logger.debug( + "managed_id_rewriter: raw object-id guard lookup failed", exc_info=True + ) + return + if existing is not None and not can_access_resource( + user_api_key_dict, existing.created_by, existing.team_id + ): + raise HTTPException(status_code=404, detail="Managed resource not found.") + + +# --------------------------------------------------------------------------- +# OUTPUT path — helpers for minting and storing managed IDs +# --------------------------------------------------------------------------- + + +def _build_managed_file_object( + snapshot: Optional[Dict[str, Any]], managed_id: str +) -> Optional[OpenAIFileObject]: + """Build an ``OpenAIFileObject`` (with the managed ID swapped in) from an + upstream file response so the DB-served list returns the same metadata as a + direct file GET. Returns ``None`` when no usable snapshot is available, in + which case the row is stored without metadata (previous behaviour).""" + if not snapshot: + return None + try: + return OpenAIFileObject(**{**snapshot, "id": managed_id}) + except Exception: + verbose_proxy_logger.debug( + "managed_id_rewriter: file object snapshot incomplete; " + "storing file row without list metadata", + exc_info=True, + ) + return None + + +async def _mint_or_reuse_file( + raw_id: str, + provider: str, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: Any, + managed_files_hook: Any, + file_object_snapshot: Optional[Dict[str, Any]] = None, + is_create_route: bool = True, +) -> str: + """Return an existing managed file ID or mint + store a new one.""" + if prisma_client is None and managed_files_hook is None: + return raw_id # no persistence available; leave raw + + # Dedup + cross-tenant guard. Look up existing passthrough rows for this + # raw id WITHOUT scoping to the caller, so a raw file id that belongs to a + # different tenant is denied rather than re-minted under the caller. A raw + # id only reaches this OUTPUT path by skipping the managed-id input gate (raw + # provider ids are opt-out), so a row owned by someone else means the caller + # is touching another tenant's upstream file. flat_model_file_ids uses array + # containment (no index, acceptable at the scale managed-file features run). + # + # The file table has no provider column, so the same raw id can map to one + # row per provider (OpenAI and Azure both use the ``file-`` format). Fetch + # all matches and filter to this provider in the application layer, picking + # the oldest match deterministically so two providers issuing the same raw id + # reuse a stable row instead of minting duplicate rows on every call. + if prisma_client is not None: + try: + candidates = await prisma_client.db.litellm_managedfiletable.find_many( + where={"flat_model_file_ids": {"has": raw_id}}, + order={"created_at": "asc"}, + ) + except Exception: + candidates = [] + verbose_proxy_logger.debug( + "managed_id_rewriter: file dedup lookup failed", exc_info=True + ) + provider_rows = [ + row + for row in (candidates or []) + if _managed_id_matches_provider(row.unified_file_id, provider) + ] + owned_row = next( + ( + row + for row in provider_rows + if can_access_resource(user_api_key_dict, row.created_by, row.team_id) + ), + None, + ) + if owned_row is not None: + verbose_proxy_logger.debug( + "managed_id_rewriter: reusing existing managed file id for raw prefix=%s", + raw_id.split("-", 1)[0], + ) + return owned_row.unified_file_id + if provider_rows: + if not is_create_route: + # Retrieve / delete: the caller supplied another owner's raw file + # id, so deny instead of minting a fresh managed id that would + # grant them cross-tenant access. + raise HTTPException( + status_code=404, + detail="Managed resource not found.", + ) + # Create only: the caller's own upstream upload reused a raw id a + # different owner already holds (two upstream accounts under one + # provider name); the file is the caller's, so leave it unmanaged. + verbose_proxy_logger.debug( + "managed_id_rewriter: file dedup hit different owner on create; " + "leaving raw id unmanaged for prefix=%s", + raw_id.split("-", 1)[0], + ) + return raw_id + + # No existing row — mint a new managed ID and store it. + managed_id = new_managed_id(provider, raw_id) + verbose_proxy_logger.debug( + "managed_id_rewriter: minted new managed file id for raw prefix=%s", + raw_id.split("-", 1)[0], + ) + if managed_files_hook is not None: + try: + await managed_files_hook.store_unified_file_id( + file_id=managed_id, + file_object=_build_managed_file_object( + file_object_snapshot, managed_id + ), + litellm_parent_otel_span=None, + model_mappings={ + _passthrough_sentinel_model_id(provider): raw_id, + _PASSTHROUGH_PROVIDER_MARKER_KEY: _passthrough_provider_marker( + provider + ), + }, + user_api_key_dict=user_api_key_dict, + ) + except Exception: + # No row backs the minted ID, so every later resolve would 404. Fall + # back to the raw id (as when no persistence is available) to keep the + # caller's freshly-created resource reachable rather than orphaned. + verbose_proxy_logger.warning( + "managed_id_rewriter: could not persist file row; " + "leaving raw id unmanaged", + exc_info=True, + ) + return raw_id + return managed_id + + +async def _mint_or_reuse_object( + raw_id: str, + provider: str, + file_purpose: str, + body_snapshot: dict, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: Any, + is_create_route: bool, +) -> str: + """Return an existing managed object ID (batch/response) or mint + store one.""" + if prisma_client is None: + return raw_id + + # Namespace raw_id with provider so two providers that happen to issue + # the same raw batch/response ID get distinct rows. The @unique constraint + # on model_object_id would otherwise cause a UniqueConstraintViolation when + # the second provider tries to insert, silently losing the persisted mapping + # and causing every subsequent _resolve_one for that ID to return 404. + # This mirrors the pattern in container_endpoints/ownership.py which uses + # f"{purpose}:{provider}:{raw_id}" for the same reason. + namespaced_model_object_id = f"passthrough:{provider}:{raw_id}" + + async def _reuse_existing(existing: Any, refresh_snapshot: bool) -> str: + """Resolve an already-persisted namespaced row: enforce the access + check, optionally refresh the snapshot, and return its managed ID.""" + if not can_access_resource( + user_api_key_dict, existing.created_by, existing.team_id + ): + if not is_create_route: + # Retrieve / cancel / delete: the caller supplied a raw ID whose + # managed row belongs to someone else. A raw ID only reaches the + # upstream by bypassing the managed-ID input gate, so deny here + # instead of echoing another owner's object back to the caller. + raise HTTPException( + status_code=404, + detail="Managed resource not found.", + ) + # Create only: the caller's upstream create just succeeded under a + # raw id a different owner already holds (two upstream accounts under + # one provider name). The object is the caller's own, so leave the raw + # id unmanaged rather than 404 a successful create; a new row can't be + # minted because model_object_id is @unique. + verbose_proxy_logger.debug( + "managed_id_rewriter: object dedup hit different owner on create; " + "leaving raw id unmanaged for prefix=%s", + raw_id.split("_", 1)[0], + ) + return raw_id + if refresh_snapshot: + # Refresh the stored snapshot so DB-served list responses reflect + # the batch's latest state (e.g. output_file_id / error_file_id that + # were null at creation but populated once the batch completed). + try: + await prisma_client.db.litellm_managedobjecttable.update( + where={"unified_object_id": existing.unified_object_id}, + data={ + "file_object": json.dumps(body_snapshot), + "updated_by": user_api_key_dict.user_id, + }, + ) + except Exception: + verbose_proxy_logger.debug( + "managed_id_rewriter: object snapshot refresh failed", + exc_info=True, + ) + verbose_proxy_logger.debug( + "managed_id_rewriter: reusing existing managed object id for raw prefix=%s", + raw_id.split("_", 1)[0], + ) + return existing.unified_object_id + + # Dedup: look up by the namespaced key — guaranteed unique per provider. + try: + existing = await prisma_client.db.litellm_managedobjecttable.find_first( + where={"model_object_id": namespaced_model_object_id} + ) + except Exception: + verbose_proxy_logger.debug( + "managed_id_rewriter: object dedup lookup failed", exc_info=True + ) + existing = None + + if existing is not None: + return await _reuse_existing(existing, refresh_snapshot=True) + + # No existing row — mint and upsert. + managed_id = new_managed_id(provider, raw_id) + verbose_proxy_logger.debug( + "managed_id_rewriter: minted new managed object id for raw prefix=%s", + raw_id.split("_", 1)[0], + ) + try: + await prisma_client.db.litellm_managedobjecttable.upsert( + where={"unified_object_id": managed_id}, + data={ + "create": { + "unified_object_id": managed_id, + "file_object": json.dumps(body_snapshot), + "model_object_id": namespaced_model_object_id, + "file_purpose": file_purpose, + "created_by": user_api_key_dict.user_id, + "team_id": user_api_key_dict.team_id, + "updated_by": user_api_key_dict.user_id, + }, + "update": { + "updated_by": user_api_key_dict.user_id, + }, + }, + ) + except Exception: + # A concurrent caller may have inserted the same namespaced row between + # our dedup lookup and this insert (model_object_id is @unique, so the + # loser's create hits a UniqueConstraintViolation). Re-read it and reuse + # the winner's managed ID so both callers converge on one ID instead of + # the loser silently keeping the raw id. + try: + raced = await prisma_client.db.litellm_managedobjecttable.find_first( + where={"model_object_id": namespaced_model_object_id} + ) + except Exception: + raced = None + if raced is not None: + return await _reuse_existing(raced, refresh_snapshot=False) + # No row backs the minted ID, so every later resolve would 404. Fall + # back to the raw id (as when no persistence is available) to keep the + # caller's freshly-created resource reachable rather than orphaned. + verbose_proxy_logger.warning( + "managed_id_rewriter: could not persist object row; " + "leaving raw id unmanaged", + exc_info=True, + ) + return raw_id + return managed_id + + +async def rewrite_response_ids( + provider: str, + method: str, + route: str, + body: dict, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: Any, + managed_files_hook: Any, +) -> dict: + """ + Mint managed IDs for raw provider values listed in + ``BUILTIN_OUTPUT_ID_FIELD_MAP`` and swap them into *body*. + + Returns the same *body* object (unchanged) when no map entry exists for + this ``(provider, method, route)`` combination. + Returns a shallow-copy of *body* with swapped values when any field is + rewritten. + """ + from litellm.proxy.auth.auth_utils import normalize_request_route + + # Strip passthrough prefix then normalize to get e.g. /v1/batches/{batch_id} + canonical = normalize_request_route(_canonical_path(route)) + field_specs = BUILTIN_OUTPUT_ID_FIELD_MAP.get((provider, method, canonical)) + if field_specs is None: + verbose_proxy_logger.debug( + "managed_id_rewriter: no output rewrite map for provider=%s method=%s route=%s", + provider, + method, + canonical, + ) + return body + + # Collection endpoints (POST /v1/batches, /v1/responses) carry no resource + # id in the path; everything else (retrieve / cancel / delete) does. Only + # creates may degrade to a raw id on a cross-owner collision. + is_create_route = "{" not in canonical + + mutated = dict(body) # shallow copy; only return if something changed + changed = False + + def _record(field_name: str, raw_value: str, managed_id: str) -> None: + nonlocal changed + if managed_id != raw_value: + mutated[field_name] = managed_id + changed = True + verbose_proxy_logger.debug( + "managed_id_rewriter: output field rewritten field=%s route=%s method=%s", + field_name, + canonical, + method, + ) + + # File fields are rewritten first so that nested references (e.g. a batch's + # input_file_id) are already managed IDs when the object snapshot is + # captured below — keeping the DB-served list in sync with a direct GET. + for field_name, expected_prefix in field_specs: + if expected_prefix not in _FILE_PREFIXES: + continue + raw_value = mutated.get(field_name) + if not isinstance(raw_value, str) or not raw_value.startswith(expected_prefix): + continue + managed_id = await _mint_or_reuse_file( + raw_value, + provider, + user_api_key_dict, + prisma_client, + managed_files_hook, + # The file's own ``id`` carries the full upstream metadata; nested + # references do not, so only the former is persisted as a snapshot. + file_object_snapshot=body if field_name == "id" else None, + is_create_route=is_create_route, + ) + _record(field_name, raw_value, managed_id) + + for field_name, expected_prefix in field_specs: + if expected_prefix in _FILE_PREFIXES: + continue + raw_value = mutated.get(field_name) + if not isinstance(raw_value, str) or not raw_value.startswith(expected_prefix): + continue + purpose = "batch" if raw_value.startswith("batch_") else "response" + managed_id = await _mint_or_reuse_object( + raw_value, + provider, + purpose, + mutated, + user_api_key_dict, + prisma_client, + is_create_route, + ) + _record(field_name, raw_value, managed_id) + + verbose_proxy_logger.debug( + "managed_id_rewriter: output rewrite completed changed=%s provider=%s method=%s route=%s", + changed, + provider, + method, + canonical, + ) + return mutated if changed else body + + +# --------------------------------------------------------------------------- +# List-route interception — serve listing entirely from DB +# --------------------------------------------------------------------------- + + +def is_passthrough_list_route(provider: str, method: str, route: str) -> bool: + """Return True when this is a GET list route whose results should be served + from the DB (user-scoped) rather than forwarded upstream.""" + if method != "GET": + return False + from litellm.proxy.auth.auth_utils import normalize_request_route + + canonical = normalize_request_route(_canonical_path(route)) + return (provider, canonical) in _LIST_ROUTE_TABLE + + +def _parse_file_object(file_object: Any) -> Any: + """Prisma may return ``Json`` columns as either a parsed dict or the raw + JSON string (depending on driver / row source). Mirror the handling used + elsewhere (see ``openai_files_endpoints/common_utils.py``) so callers can + treat the result uniformly. + """ + if isinstance(file_object, str): + try: + return json.loads(file_object) + except (TypeError, ValueError): + return None + return file_object + + +def _empty_list_response() -> Dict[str, Any]: + return { + "object": "list", + "data": [], + "first_id": None, + "last_id": None, + "has_more": False, + } + + +def _parse_list_limit(query_params: Optional[Dict[str, Any]]) -> Tuple[int, int]: + params = query_params or {} + try: + raw_limit = int(params.get("limit", 20)) + except (TypeError, ValueError): + raw_limit = 20 + # Fetch one extra to cheaply detect has_more. + return raw_limit, min(raw_limit, 100) + 1 + + +async def _build_list_where_with_cursor( + prisma_client: Any, + resource_kind: str, + provider: str, + owner_filter: Dict[str, Any], + query_params: Optional[Dict[str, Any]], +) -> Tuple[Dict[str, Any], str]: + """Return a Prisma ``where`` clause and fetch order for a list query.""" + params = query_params or {} + after_id: Optional[str] = params.get("after") + before_id: Optional[str] = params.get("before") + where: Dict[str, Any] = dict(owner_filter) + fetch_order = "desc" + + cursor_id = after_id or before_id + # A cursor minted for a different provider would resolve to that provider's + # created_at boundary and silently skip/repeat this provider's rows, so + # ignore it and serve the unscoped first page instead. + if not cursor_id or not _managed_id_matches_provider(cursor_id, provider): + return where, fetch_order + + cursor_table = ( + prisma_client.db.litellm_managedfiletable + if resource_kind == "files" + else prisma_client.db.litellm_managedobjecttable + ) + cursor_field = ( + "unified_file_id" if resource_kind == "files" else "unified_object_id" + ) + try: + cursor_row = await cursor_table.find_first( + where={**owner_filter, cursor_field: cursor_id} + ) + if cursor_row is not None: + if after_id: + op = "lt" + else: + op = "gt" + fetch_order = "asc" + # created_at is not unique, so the boundary must also compare the + # unique id (the secondary sort key) to avoid skipping or repeating + # rows that share the cursor row's timestamp across a page boundary. + boundary = { + "OR": [ + {"created_at": {op: cursor_row.created_at}}, + { + "AND": [ + {"created_at": cursor_row.created_at}, + {cursor_field: {op: cursor_id}}, + ] + }, + ] + } + where = {"AND": [where, boundary]} if where else boundary + except Exception: + pass + return where, fetch_order + + +async def _fetch_list_rows( + prisma_client: Any, + resource_kind: str, + where: Dict[str, Any], + fetch_order: str, + fetch_limit: int, +) -> Optional[List[Any]]: + # created_at is not unique, so a second sort on the unique id column gives a + # total order, keeping the limit+1 page boundary and cursor deterministic + # across rows that share a created_at timestamp. + try: + if resource_kind == "files": + return await prisma_client.db.litellm_managedfiletable.find_many( + where=where, + order=[{"created_at": fetch_order}, {"unified_file_id": fetch_order}], + take=fetch_limit, + ) + return await prisma_client.db.litellm_managedobjecttable.find_many( + where={**where, "file_purpose": "batch"}, + order=[{"created_at": fetch_order}, {"unified_object_id": fetch_order}], + take=fetch_limit, + ) + except Exception: + verbose_proxy_logger.warning( + "managed_id_rewriter: list DB query failed", exc_info=True + ) + return None + + +async def _fetch_provider_scoped_list_rows( + prisma_client: Any, + resource_kind: str, + provider: str, + where: Dict[str, Any], + fetch_order: str, + raw_limit: int, + fetch_limit: int, +) -> Tuple[List[Any], bool]: + """Fetch one page of list rows scoped to *provider* at the DB level. + + Both resource kinds carry a provider-distinguishing value that the query + filters on directly: object rows namespace ``model_object_id`` as + ``passthrough:{provider}:{raw}`` (see ``_mint_or_reuse_object``) and file + rows carry ``_passthrough_provider:{provider}`` in ``flat_model_file_ids`` + (see ``_mint_or_reuse_file``), since the file table has no provider column. + Pushing the scope into the query means a single DB round-trip serves the + page, with no application-layer scanning that could truncate large pools. + + A DB failure returns an empty page (fail closed) so the caller never falls + through to the upstream provider. + """ + scoped_where = dict(where) + if resource_kind == "files": + scoped_where["flat_model_file_ids"] = { + "has": _passthrough_provider_marker(provider) + } + else: + scoped_where["model_object_id"] = {"startswith": f"passthrough:{provider}:"} + + rows = await _fetch_list_rows( + prisma_client, resource_kind, scoped_where, fetch_order, fetch_limit + ) + if rows is None: + return [], False + + effective_limit = min(raw_limit, 100) + has_more = len(rows) > effective_limit + page = rows[:effective_limit] + if fetch_order == "asc": + page = list(reversed(page)) + return page, has_more + + +def _serialize_file_list_item(row: Any) -> Dict[str, Any]: + item: Dict[str, Any] = { + "id": row.unified_file_id, + "object": "file", + "created_at": int(row.created_at.timestamp()) if row.created_at else None, + } + file_object = _parse_file_object(row.file_object) + if isinstance(file_object, dict): + item.update(file_object) + item["id"] = row.unified_file_id # managed ID always wins over stored raw id + return item + + +def _serialize_batch_list_item(row: Any) -> Dict[str, Any]: + item: Dict[str, Any] = {} + file_object = _parse_file_object(row.file_object) + if isinstance(file_object, dict): + item.update(file_object) + item["id"] = row.unified_object_id # managed ID always wins + item["object"] = "batch" + return item + + +def _list_boundary_ids( + rows: List[Any], resource_kind: str +) -> Tuple[Optional[str], Optional[str]]: + if not rows: + return None, None + id_attr = "unified_file_id" if resource_kind == "files" else "unified_object_id" + return getattr(rows[0], id_attr), getattr(rows[-1], id_attr) + + +async def list_passthrough_ids_from_db( + provider: str, + route: str, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: Any, + query_params: Optional[Dict[str, Any]] = None, +) -> Optional[Dict[str, Any]]: + """Query the DB for managed IDs the caller owns and return an OpenAI-style + paginated list response. + + Returns ``None`` when ``prisma_client`` is unavailable or the route is not + a recognised list route (caller should fall through to upstream). + + Pagination params ``after``, ``before``, and ``limit`` are read from + ``query_params`` to match the OpenAI Batches / Files list API. + + Ownership scoping: + - Proxy admins / master key: see **all** rows. + - Regular users: only rows matching their ``user_id`` / ``team_id``. + """ + if prisma_client is None: + return None + + from litellm.proxy.auth.auth_utils import normalize_request_route + + canonical = normalize_request_route(_canonical_path(route)) + resource_kind = _LIST_ROUTE_TABLE.get((provider, canonical)) + if resource_kind is None: + return None + + owner_filter = build_owner_filter(user_api_key_dict) + if owner_filter is None: + verbose_proxy_logger.warning( + "managed_id_rewriter: list denied — caller has no user_id or team_id" + ) + return _empty_list_response() + + raw_limit, fetch_limit = _parse_list_limit(query_params) + where, fetch_order = await _build_list_where_with_cursor( + prisma_client, resource_kind, provider, owner_filter, query_params + ) + page, has_more = await _fetch_provider_scoped_list_rows( + prisma_client, + resource_kind, + provider, + where, + fetch_order, + raw_limit, + fetch_limit, + ) + if resource_kind == "files": + data = [_serialize_file_list_item(row) for row in page] + else: + data = [_serialize_batch_list_item(row) for row in page] + + first_id, last_id = _list_boundary_ids(page, resource_kind) + verbose_proxy_logger.debug( + "managed_id_rewriter: list served from DB provider=%s kind=%s count=%d admin=%s", + provider, + resource_kind, + len(data), + owner_filter == {}, + ) + return { + "object": "list", + "data": data, + "first_id": first_id, + "last_id": last_id, + "has_more": has_more, + } + + +# --------------------------------------------------------------------------- +# INPUT path extractors — all delegate to _resolve_one +# --------------------------------------------------------------------------- + + +async def rewrite_path_ids( + path: str, + provider: str, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: Any, + managed_files_hook: Any, +) -> str: + """ + Walk URL path segments and resolve any passthrough managed IDs to raw + provider IDs. Returns *path* unchanged when no managed IDs are found. + """ + budget = _RawIdGuardBudget() + segments = path.split("/") + new_segments: List[str] = [] + changed = False + for seg in segments: + decoded_seg = unquote(seg) + if is_managed(decoded_seg): + raw = await _resolve_one( + decoded_seg, + provider, + user_api_key_dict, + prisma_client, + managed_files_hook, + ) + new_segments.append(quote(raw, safe="-_.~")) + changed = True + else: + await _guard_raw_provider_id( + decoded_seg, provider, user_api_key_dict, prisma_client, budget + ) + new_segments.append(seg) + if changed: + verbose_proxy_logger.debug( + "managed_id_rewriter: path ids rewritten provider=%s", provider + ) + return "/".join(new_segments) if changed else path + + +async def rewrite_query_ids( + params: Optional[Dict[str, Any]], + provider: str, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: Any, + managed_files_hook: Any, +) -> Optional[Dict[str, Any]]: + """ + Walk query param values and resolve any passthrough managed IDs. + Returns *params* unchanged (same object) when nothing is resolved. + """ + if not params: + return params + budget = _RawIdGuardBudget() + mutated = dict(params) + rewritten_keys: List[str] = [] + for key, val in list(mutated.items()): + if isinstance(val, str): + if is_managed(val): + mutated[key] = await _resolve_one( + val, provider, user_api_key_dict, prisma_client, managed_files_hook + ) + rewritten_keys.append(key) + else: + await _guard_raw_provider_id( + val, provider, user_api_key_dict, prisma_client, budget + ) + if rewritten_keys: + verbose_proxy_logger.debug( + "managed_id_rewriter: query ids rewritten provider=%s keys=%s", + provider, + rewritten_keys, + ) + return mutated if rewritten_keys else params + + +async def rewrite_body_ids( + body: Optional[Dict[str, Any]], + provider: str, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: Any, + managed_files_hook: Any, +) -> Optional[Dict[str, Any]]: + """ + Recursively walk a request body dict/list and resolve any passthrough + managed IDs. Skips litellm internal keys (``litellm_*``). + Returns *body* unchanged (same object) when nothing is resolved. + """ + if not body: + return body + + budget = _RawIdGuardBudget() + + async def _walk(node: Any, depth: int) -> Any: + if depth >= _MAX_BODY_REWRITE_DEPTH: + return node + if isinstance(node, dict): + result: Dict[str, Any] = {} + changed_inner = False + for k, v in node.items(): + # Skip litellm internal injection keys (e.g. litellm_logging_obj) + if isinstance(k, str) and k.startswith("litellm_"): + result[k] = v + continue + new_v = await _walk(v, depth + 1) + result[k] = new_v + if new_v is not v: + changed_inner = True + return result if changed_inner else node + elif isinstance(node, list): + new_list = [await _walk(item, depth + 1) for item in node] + if any(n is not o for n, o in zip(new_list, node)): + return new_list + return node + elif isinstance(node, str): + if is_managed(node): + return await _resolve_one( + node, provider, user_api_key_dict, prisma_client, managed_files_hook + ) + await _guard_raw_provider_id( + node, provider, user_api_key_dict, prisma_client, budget + ) + return node + return node + + rewritten = await _walk(body, 0) + if rewritten is not body: + verbose_proxy_logger.debug( + "managed_id_rewriter: body ids rewritten provider=%s", provider + ) + return rewritten diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 5aa0f6cb18..f08e021630 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -1,3065 +1,3252 @@ -import ast -import asyncio -import copy -import json -import posixpath -import traceback -from base64 import b64encode -from datetime import datetime -from typing import Any, Dict, List, Mapping, Optional, Tuple, Union, cast -from urllib.parse import urlencode, urlparse - -import httpx -from fastapi import ( - APIRouter, - Depends, - FastAPI, - HTTPException, - Request, - Response, - UploadFile, - WebSocket, - status, -) -from fastapi.responses import StreamingResponse -from starlette.datastructures import UploadFile as StarletteUploadFile -from starlette.websockets import WebSocketState -from websockets.asyncio.client import connect -from websockets.exceptions import ( - ConnectionClosedError, - ConnectionClosedOK, - InvalidStatus, -) - -import litellm -from litellm._logging import verbose_proxy_logger -from litellm._uuid import uuid -from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG -from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.safe_json_dumps import safe_dumps -from litellm.llms.custom_httpx.http_handler import get_async_httpx_client -from litellm.passthrough import BasePassthroughUtils -from litellm.proxy._types import ( - ConfigFieldInfo, - ConfigFieldUpdate, - LiteLLMRoutes, - PassThroughEndpointResponse, - PassThroughGenericEndpoint, - ProxyException, - UserAPIKeyAuth, -) -from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.proxy.common_utils.http_parsing_utils import ( - _read_request_body, - _safe_get_request_headers, -) -from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup -from litellm.proxy.utils import get_server_root_path, normalize_route_for_root_path -from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.custom_http import httpxSpecialProvider -from litellm.types.passthrough_endpoints.pass_through_endpoints import ( - EndpointType, - LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, - LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, - PassthroughStandardLoggingPayload, -) - -from .streaming_handler import PassThroughStreamingHandler -from .success_handler import PassThroughEndpointLogging - -router = APIRouter() - -pass_through_endpoint_logging = PassThroughEndpointLogging() - -# Global registry to track registered pass-through routes and prevent memory leaks -_registered_pass_through_routes: Dict[ - str, Dict[str, Union[str, List[str], Dict[str, Any]]] -] = {} - - -def get_response_body(response: httpx.Response) -> Optional[dict]: - try: - return response.json() - except Exception: - return None - - -async def set_env_variables_in_header(custom_headers: Optional[dict]) -> Optional[dict]: - """ - checks if any headers on config.yaml are defined as os.environ/COHERE_API_KEY etc - - only runs for headers defined on config.yaml - - example header can be - - {"Authorization": "Bearer os.environ/COHERE_API_KEY"} - """ - if custom_headers is None: - return None - headers = {} - for key, value in custom_headers.items(): - # langfuse Api requires base64 encoded headers - it's simpleer to just ask litellm users to set their langfuse public and secret keys - # we can then get the b64 encoded keys here - if key == "LANGFUSE_PUBLIC_KEY" or key == "LANGFUSE_SECRET_KEY": - # langfuse requires b64 encoded headers - we construct that here - _langfuse_public_key = custom_headers["LANGFUSE_PUBLIC_KEY"] - _langfuse_secret_key = custom_headers["LANGFUSE_SECRET_KEY"] - if isinstance( - _langfuse_public_key, str - ) and _langfuse_public_key.startswith("os.environ/"): - _langfuse_public_key = get_secret_str(_langfuse_public_key) - if isinstance( - _langfuse_secret_key, str - ) and _langfuse_secret_key.startswith("os.environ/"): - _langfuse_secret_key = get_secret_str(_langfuse_secret_key) - headers["Authorization"] = "Basic " + b64encode( - f"{_langfuse_public_key}:{_langfuse_secret_key}".encode("utf-8") - ).decode("ascii") - else: - # for all other headers - headers[key] = value - if isinstance(value, str) and "os.environ/" in value: - verbose_proxy_logger.debug( - "pass through endpoint - looking up 'os.environ/' variable" - ) - # get string section that is os.environ/ - start_index = value.find("os.environ/") - _variable_name = value[start_index:] - - verbose_proxy_logger.debug( - "pass through endpoint - getting secret for variable name: %s", - _variable_name, - ) - _secret_value = get_secret_str(_variable_name) - if _secret_value is not None: - new_value = value.replace(_variable_name, _secret_value) - headers[key] = new_value - return headers - - -async def chat_completion_pass_through_endpoint( # noqa: PLR0915 - fastapi_response: Response, - request: Request, - adapter_id: str, - user_api_key_dict: UserAPIKeyAuth, -): - from litellm.proxy.proxy_server import ( - add_litellm_data_to_request, - general_settings, - llm_router, - proxy_config, - proxy_logging_obj, - user_api_base, - user_max_tokens, - user_model, - user_request_timeout, - user_temperature, - version, - ) - - data = {} - try: - body = await request.body() - body_str = body.decode() - try: - data = ast.literal_eval(body_str) - except Exception: - data = json.loads(body_str) - - data["adapter_id"] = adapter_id - - verbose_proxy_logger.debug( - "Request received by LiteLLM:\n{}".format(json.dumps(data, indent=4)), - ) - data["model"] = ( - general_settings.get("completion_model", None) # server default - or user_model # model name passed via cli args - or data.get("model", None) # default passed in http request - ) - if user_model: - data["model"] = user_model - - data = await add_litellm_data_to_request( - data=data, # type: ignore - request=request, - general_settings=general_settings, - user_api_key_dict=user_api_key_dict, - version=version, - proxy_config=proxy_config, - ) - - # override with user settings, these are params passed via cli - if user_temperature: - data["temperature"] = user_temperature - if user_request_timeout: - data["request_timeout"] = user_request_timeout - if user_max_tokens: - data["max_tokens"] = user_max_tokens - if user_api_base: - data["api_base"] = user_api_base - - ### MODEL ALIAS MAPPING ### - # check if model name in model alias map - # get the actual model name - if data["model"] in litellm.model_alias_map: - data["model"] = litellm.model_alias_map[data["model"]] - - # Check key-specific aliases - if ( - isinstance(data["model"], str) - and user_api_key_dict.aliases - and isinstance(user_api_key_dict.aliases, dict) - and data["model"] in user_api_key_dict.aliases - ): - data["model"] = user_api_key_dict.aliases[data["model"]] - - ### CALL HOOKS ### - modify incoming data before calling the model - data = await proxy_logging_obj.pre_call_hook( # type: ignore - user_api_key_dict=user_api_key_dict, data=data, call_type="text_completion" - ) - - ### ROUTE THE REQUESTs ### - router_model_names = llm_router.model_names if llm_router is not None else [] - # skip router if user passed their key - if "api_key" in data: - llm_response = asyncio.create_task(litellm.aadapter_completion(**data)) - elif ( - llm_router is not None and data["model"] in router_model_names - ): # model in router model list - llm_response = asyncio.create_task(llm_router.aadapter_completion(**data)) - elif ( - llm_router is not None - and llm_router.model_group_alias is not None - and data["model"] in llm_router.model_group_alias - ): # model set in model_group_alias - llm_response = asyncio.create_task(llm_router.aadapter_completion(**data)) - elif llm_router is not None and llm_router.has_model_id( - data["model"] - ): # model in router model list - llm_response = asyncio.create_task(llm_router.aadapter_completion(**data)) - elif ( - llm_router is not None - and data["model"] not in router_model_names - and ( - llm_router.default_deployment is not None - or len(llm_router.pattern_router.patterns) > 0 - ) - ): # check for wildcard routes or default deployment before checking deployment_names - llm_response = asyncio.create_task(llm_router.aadapter_completion(**data)) - elif ( - llm_router is not None and data["model"] in llm_router.deployment_names - ): # model in router deployments, calling a specific deployment on the router (lowest priority) - llm_response = asyncio.create_task( - llm_router.aadapter_completion(**data, specific_deployment=True) - ) - elif user_model is not None: # `litellm --model ` - llm_response = asyncio.create_task(litellm.aadapter_completion(**data)) - else: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail={ - "error": "completion: Invalid model name passed in model=" - + data.get("model", "") - }, - ) - - # Await the llm_response task - response = await llm_response - - hidden_params = getattr(response, "_hidden_params", {}) or {} - model_id = hidden_params.get("model_id", None) or "" - cache_key = hidden_params.get("cache_key", None) or "" - api_base = hidden_params.get("api_base", None) or "" - response_cost = hidden_params.get("response_cost", None) or "" - - ### ALERTING ### - asyncio.create_task( - proxy_logging_obj.update_request_status( - litellm_call_id=data.get("litellm_call_id", ""), status="success" - ) - ) - - verbose_proxy_logger.debug("final response: %s", response) - - fastapi_response.headers.update( - ProxyBaseLLMRequestProcessing.get_custom_headers( - user_api_key_dict=user_api_key_dict, - model_id=model_id, - cache_key=cache_key, - api_base=api_base, - version=version, - response_cost=response_cost, - ) - ) - - verbose_proxy_logger.debug("\nResponse from Litellm:\n{}".format(response)) - return response - except Exception as e: - await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data - ) - verbose_proxy_logger.exception( - "litellm.proxy.proxy_server.completion(): Exception occured - {}".format( - str(e) - ) - ) - error_msg = f"{str(e)}" - raise ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", 500), - ) - - -class HttpPassThroughEndpointHelpers(BasePassthroughUtils): - @staticmethod - def get_response_headers( - headers: httpx.Headers, - litellm_call_id: Optional[str] = None, - custom_headers: Optional[dict] = None, - ) -> dict: - # Exclude headers that uvicorn writes itself (server, date) and - # encoding/length headers that don't survive re-serialization. - # If we forward the upstream's Server header, uvicorn adds its - # own and strict HTTP parsers (e.g. aiohttp) reject the - # response with "Duplicate 'Server' header found". - excluded_headers = { - "transfer-encoding", - "content-encoding", - "content-length", - "server", - "date", - "connection", - "keep-alive", - } - - return_headers = { - key: value - for key, value in headers.items() - if key.lower() not in excluded_headers - } - if litellm_call_id: - return_headers["x-litellm-call-id"] = litellm_call_id - if custom_headers: - # Ensure custom headers don't override actual upstream response headers or let framework defaults (like content-length: 0) interfere. - sanitized_custom_headers = { - key: value - for key, value in custom_headers.items() - if key.lower() not in excluded_headers - } - return_headers.update(sanitized_custom_headers) - - return return_headers - - @staticmethod - def get_endpoint_type(url: str) -> EndpointType: - parsed_url = urlparse(url) - if ( - ("generateContent") in url - or ("streamGenerateContent") in url - or ("rawPredict") in url - or ("streamRawPredict") in url - ): - return EndpointType.VERTEX_AI - elif parsed_url.hostname == "api.anthropic.com": - return EndpointType.ANTHROPIC - elif ( - parsed_url.hostname == "api.openai.com" - or parsed_url.hostname == "openai.azure.com" - or (parsed_url.hostname and "openai.com" in parsed_url.hostname) - ): - return EndpointType.OPENAI - return EndpointType.GENERIC - - @staticmethod - async def _make_non_streaming_http_request( - request: Request, - async_client: httpx.AsyncClient, - url: str, - headers: dict, - requested_query_params: Optional[dict] = None, - custom_body: Optional[dict] = None, - ) -> httpx.Response: - """ - Make a non-streaming HTTP request - - If request is GET, don't include a JSON body - """ - if request.method == "GET": - response = await async_client.request( - method=request.method, - url=url, - headers=headers, - params=requested_query_params, - ) - else: - response = await async_client.request( - method=request.method, - url=url, - headers=headers, - params=requested_query_params, - json=custom_body, - ) - return response - - @staticmethod - async def non_streaming_http_request_handler( - request: Request, - async_client: httpx.AsyncClient, - url: httpx.URL, - headers: dict, - requested_query_params: Optional[dict] = None, - _parsed_body: Optional[dict] = None, - forward_multipart: bool = False, - ) -> httpx.Response: - """ - Handle non-streaming HTTP requests - - Handles special cases when GET requests, multipart/form-data requests, and generic httpx requests - """ - if request.method == "GET": - response = await async_client.request( - method=request.method, - url=url, - headers=headers, - params=requested_query_params, - ) - elif ( - HttpPassThroughEndpointHelpers.is_multipart(request) is True - and forward_multipart - ): - # Forward multipart via make_multipart_http_request even when _parsed_body is - # non-empty (pass_through_request always injects litellm_logging_obj, etc.). - # forward_multipart is False when custom_body was supplied (JSON body despite - # multipart content-type) — those requests use the generic json= path. - return await HttpPassThroughEndpointHelpers.make_multipart_http_request( - request=request, - async_client=async_client, - url=url, - headers=headers, - requested_query_params=requested_query_params, - ) - else: - # Generic httpx method - response = await async_client.request( - method=request.method, - url=url, - headers=headers, - params=requested_query_params, - json=_parsed_body, - ) - return response - - @staticmethod - def is_multipart(request: Request) -> bool: - """Check if the request is a multipart/form-data request""" - return "multipart/form-data" in request.headers.get("content-type", "") - - @staticmethod - async def _build_request_files_from_upload_file( - upload_file: Union[UploadFile, StarletteUploadFile], - ) -> Tuple[Optional[str], bytes, Optional[str]]: - """Build a request files dict from an UploadFile object""" - file_content = await upload_file.read() - return (upload_file.filename, file_content, upload_file.content_type) - - @staticmethod - async def make_multipart_http_request( - request: Request, - async_client: httpx.AsyncClient, - url: httpx.URL, - headers: dict, - requested_query_params: Optional[dict] = None, - stream: bool = False, - ) -> httpx.Response: - """Process multipart/form-data requests, handling both files and form fields""" - form_data = await request.form() - files = {} - form_data_dict = {} - - for field_name, field_value in form_data.items(): - if isinstance(field_value, (StarletteUploadFile, UploadFile)): - files[field_name] = ( - await HttpPassThroughEndpointHelpers._build_request_files_from_upload_file( - upload_file=field_value - ) - ) - else: - form_data_dict[field_name] = field_value - - # Remove content-type header - httpx will set it correctly with the new boundary - # when it creates the multipart body from files/data parameters - headers_copy = headers.copy() - headers_copy.pop("content-type", None) - - # httpx.AsyncClient.request() does not accept stream=; use send() for streaming. - if stream: - req = async_client.build_request( - request.method, - url, - headers=headers_copy, - params=requested_query_params, - files=files, - data=form_data_dict, - ) - return await async_client.send(req, stream=True) - - return await async_client.request( - method=request.method, - url=url, - headers=headers_copy, - params=requested_query_params, - files=files, - data=form_data_dict, - ) - - @staticmethod - def _init_kwargs_for_pass_through_endpoint( - request: Request, - user_api_key_dict: UserAPIKeyAuth, - passthrough_logging_payload: PassthroughStandardLoggingPayload, - logging_obj: LiteLLMLoggingObj, - _parsed_body: Optional[dict] = None, - litellm_call_id: Optional[str] = None, - ) -> dict: - """ - Filter out litellm params from the request body - """ - from litellm.types.utils import all_litellm_params - - _parsed_body = _parsed_body or {} - - litellm_params_in_body = {} - for k in all_litellm_params: - if k in _parsed_body: - litellm_params_in_body[k] = _parsed_body.pop(k, None) - - _metadata = dict( - LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( - user_api_key_dict=user_api_key_dict - ) - ) - - litellm_metadata = litellm_params_in_body.pop("litellm_metadata", None) - metadata = litellm_params_in_body.pop("metadata", None) - if litellm_metadata: - _metadata.update(litellm_metadata) - if metadata: - _metadata.update(metadata) - - _metadata = _update_metadata_with_tags_in_header( - request=request, - metadata=_metadata, - ) - - # Set internal keys after merging client-supplied metadata so a request - # body that mirrors them cannot clobber the authenticated key or the - # real parent span. - _metadata["user_api_key"] = user_api_key_dict.api_key - _metadata["litellm_parent_otel_span"] = user_api_key_dict.parent_otel_span - - kwargs = { - "litellm_params": { - **litellm_params_in_body, # type: ignore - "metadata": _metadata, - "proxy_server_request": { - "url": str(request.url), - "method": request.method, - "body": copy.copy(_parsed_body), # use copy instead of deepcopy - "headers": request.headers, - }, - }, - "call_type": "pass_through_endpoint", - "litellm_call_id": litellm_call_id, - "passthrough_logging_payload": passthrough_logging_payload, - } - - logging_obj.model_call_details["passthrough_logging_payload"] = ( - passthrough_logging_payload - ) - - return kwargs - - @staticmethod - def construct_target_url_with_subpath( - base_target: str, subpath: str, include_subpath: Optional[bool] - ) -> str: - """ - Helper function to construct the full target URL with subpath handling. - - Args: - base_target: The base target URL - subpath: The captured subpath from the request - include_subpath: Whether to include the subpath in the target URL - - Returns: - The constructed full target URL - """ - if not include_subpath: - return base_target - - if not subpath: - return base_target - - # Ensure base_target ends with / and subpath doesn't start with / - if not base_target.endswith("/"): - base_target = base_target + "/" - if subpath.startswith("/"): - subpath = subpath[1:] - - # Resolve any '..' segments in the subpath so it cannot climb above - # the base_target prefix that the operator configured. Preserve a - # trailing slash on the original subpath since some upstreams treat - # `/foo` and `/foo/` as different resources. - trailing_slash = subpath.endswith("/") - safe_subpath = posixpath.normpath("/" + subpath).lstrip("/") - if safe_subpath == ".": - safe_subpath = "" - if trailing_slash and safe_subpath and not safe_subpath.endswith("/"): - safe_subpath += "/" - - return base_target + safe_subpath - - @staticmethod - def join_base_and_endpoint_path(base_url: httpx.URL, endpoint_path: str) -> str: - """ - Combine the path component of ``base_url`` with ``endpoint_path``. - - Preserves any path prefix configured on the base URL and resolves - ``..`` segments in the endpoint so the result stays within the base - path. A trailing slash on ``endpoint_path`` is preserved. - """ - trailing_slash = endpoint_path.endswith("/") - base_path = base_url.path or "" - if not base_path or base_path == "/": - normalized_endpoint = posixpath.normpath("/" + endpoint_path.lstrip("/")) - if trailing_slash and normalized_endpoint != "/": - normalized_endpoint += "/" - return normalized_endpoint - - base_path = base_path.rstrip("/") - clean_endpoint = endpoint_path.lstrip("/") - combined = posixpath.normpath(base_path + "/" + clean_endpoint) - # If normalization climbs out of the base path, fall back to base. - if combined != base_path and not combined.startswith(base_path + "/"): - return base_path + "/" - if trailing_slash and not combined.endswith("/"): - combined += "/" - return combined - - @staticmethod - def _update_stream_param_based_on_request_body( - parsed_body: dict, - stream: Optional[bool] = None, - ) -> Optional[bool]: - """ - If stream is provided in the request body, use it. - Otherwise, use the stream parameter passed to the `pass_through_request` function - """ - if "stream" in parsed_body: - return parsed_body.get("stream", stream) - return stream - - -async def pass_through_request( # noqa: PLR0915 - request: Request, - target: str, - custom_headers: dict, - user_api_key_dict: UserAPIKeyAuth, - custom_body: Optional[dict] = None, - forward_headers: Optional[bool] = False, - merge_query_params: Optional[bool] = False, - query_params: Optional[dict] = None, - default_query_params: Optional[dict] = None, - stream: Optional[bool] = None, - cost_per_request: Optional[float] = None, - custom_llm_provider: Optional[str] = None, - guardrails_config: Optional[dict] = None, -): - """ - Pass through endpoint handler, makes the httpx request for pass-through endpoints and ensures logging hooks are called - - Args: - request: The incoming request - target: The target URL - custom_headers: The custom headers - user_api_key_dict: The user API key dictionary - custom_body: The custom body - forward_headers: Whether to forward headers - merge_query_params: Whether to merge query params - query_params: The query params - default_query_params: The default query params to be applied if not overridden by client - stream: Whether to stream the response - cost_per_request: Optional field - cost per request to the target endpoint - custom_llm_provider: Optional field - custom LLM provider for the endpoint - guardrails_config: Optional field - guardrails configuration for passthrough endpoint - """ - from litellm.exceptions import ModifyResponseException - from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.proxy.pass_through_endpoints.passthrough_guardrails import ( - PassthroughGuardrailHandler, - ) - from litellm.proxy.proxy_server import proxy_logging_obj - - ######################################################### - # Initialize variables - ######################################################### - litellm_call_id = str(uuid.uuid4()) - url: Optional[httpx.URL] = None - - # parsed request body - _parsed_body: Optional[dict] = None - # kwargs for pass through endpoint, contains metadata, litellm_params, call_type, litellm_call_id, passthrough_logging_payload - kwargs: Optional[dict] = None - logging_obj: Optional[Logging] = None - - ######################################################### - try: - url = httpx.URL(target) - headers = custom_headers - headers = HttpPassThroughEndpointHelpers.forward_headers_from_request( - request_headers=_safe_get_request_headers(request).copy(), - headers=headers, - forward_headers=forward_headers, - ) - - # Apply default query parameters if provided, regardless of merge_query_params setting - if default_query_params or merge_query_params: - # Determine what to merge based on settings - request_params = dict(request.query_params) if merge_query_params else {} - - # Create a new URL with the merged query params - url = url.copy_with( - query=urlencode( - HttpPassThroughEndpointHelpers.get_merged_query_parameters( - existing_url=url, - request_query_params=request_params, - default_query_params=default_query_params, - ) - ).encode("ascii") - ) - - endpoint_type: EndpointType = HttpPassThroughEndpointHelpers.get_endpoint_type( - str(url) - ) - - # SigV4-signed callers (e.g. Bedrock) attach the exact bytes that were - # signed via request.state; we must send those instead of re-encoding the - # parsed dict (hooks mutate it, breaking the signature / Content-Length). - # Tolerate request objects without `state` (test fixtures) and only honor - # values httpx accepts for `content=`. - _request_state = getattr(request, "state", None) - state_raw_body: Optional[Union[str, bytes]] = ( - getattr(_request_state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, None) - if _request_state is not None - else None - ) - if state_raw_body is not None and not isinstance( - state_raw_body, (str, bytes, bytearray) - ): - state_raw_body = None - - # Skip body parsing for multipart requests - make_multipart_http_request will handle it - # But if custom_body is provided (e.g., JSON parsed despite multipart content-type), use it - is_multipart = ( - HttpPassThroughEndpointHelpers.is_multipart(request) and not custom_body - ) - - if custom_body: - _parsed_body = custom_body - elif is_multipart: - # Don't parse multipart body here - it will be handled by make_multipart_http_request - _parsed_body = {} - else: - _parsed_body = await _read_request_body(request) - verbose_proxy_logger.debug( - "Pass through endpoint sending request to \nURL {}\nheaders: {}\nbody: {}\n".format( - url, headers, _parsed_body - ) - ) - - ### COLLECT GUARDRAILS FOR PASSTHROUGH ENDPOINT ### - # Passthrough endpoints are opt-in only for guardrails - # When enabled, collect guardrails from org/team/key levels + passthrough-specific - guardrails_to_run = PassthroughGuardrailHandler.collect_guardrails( - user_api_key_dict=user_api_key_dict, - passthrough_guardrails_config=guardrails_config, - ) - - # Add guardrails to metadata if any should run - if guardrails_to_run and len(guardrails_to_run) > 0: - if _parsed_body is None: - _parsed_body = {} - if "metadata" not in _parsed_body: - _parsed_body["metadata"] = {} - _parsed_body["metadata"]["guardrails"] = guardrails_to_run - verbose_proxy_logger.debug( - f"Added guardrails to passthrough request metadata: {guardrails_to_run}" - ) - - ## LOGGING OBJECT ## - initialize before pre_call_hook so guardrails can access it - # Surface the requested model (when the body carries one) so logging/spans - # read e.g. ``chat gpt-4o`` instead of ``chat unknown``. - passthrough_model = ( - _parsed_body.get("model") if isinstance(_parsed_body, dict) else None - ) or "unknown" - start_time = datetime.now() - logging_obj = Logging( - model=passthrough_model, - messages=[{"role": "user", "content": safe_dumps(_parsed_body)}], - stream=False, - call_type="pass_through_endpoint", - start_time=start_time, - litellm_call_id=litellm_call_id, - function_id="1245", - ) - - # Store passthrough guardrails config on logging_obj for field targeting - logging_obj.passthrough_guardrails_config = guardrails_config - - # Store logging_obj in data so guardrails can access it - if _parsed_body is None: - _parsed_body = {} - _parsed_body["litellm_logging_obj"] = logging_obj - - ### CALL HOOKS ### - modify incoming data / reject request before calling the model - _parsed_body = await proxy_logging_obj.pre_call_hook( - user_api_key_dict=user_api_key_dict, - data=_parsed_body, - call_type="pass_through_endpoint", - ) - async_client_obj = get_async_httpx_client( - llm_provider=httpxSpecialProvider.PassThroughEndpoint, - params={"timeout": 600}, - ) - async_client = async_client_obj.client - passthrough_logging_payload = PassthroughStandardLoggingPayload( - url=str(url), - request_body=_parsed_body, - request_method=getattr(request, "method", None), - cost_per_request=cost_per_request, - ) - kwargs = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( - user_api_key_dict=user_api_key_dict, - _parsed_body=_parsed_body, - passthrough_logging_payload=passthrough_logging_payload, - litellm_call_id=litellm_call_id, - request=request, - logging_obj=logging_obj, - ) - - # Store custom_llm_provider in kwargs and logging object if provided - if custom_llm_provider: - logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider - logging_obj.model_call_details["litellm_params"] = kwargs.get( - "litellm_params", {} - ) - - # done for supporting 'parallel_request_limiter.py' with pass-through endpoints - logging_obj.update_environment_variables( - model=passthrough_model, - user="unknown", - optional_params={}, - litellm_params=kwargs["litellm_params"], - call_type="pass_through_endpoint", - ) - logging_obj.model_call_details["litellm_call_id"] = litellm_call_id - - # combine url with query params for logging - requested_query_params: Optional[dict] = query_params or dict( - request.query_params - ) - - requested_query_params_str = None - if requested_query_params: - requested_query_params_str = "&".join( - f"{k}={v}" for k, v in requested_query_params.items() - ) - - logging_url = str(url) - if requested_query_params_str: - if "?" in str(url): - logging_url = str(url) + "&" + requested_query_params_str - else: - logging_url = str(url) + "?" + requested_query_params_str - - logging_obj.pre_call( - input=[{"role": "user", "content": safe_dumps(_parsed_body)}], - api_key="", - additional_args={ - "complete_input_dict": _parsed_body, - "api_base": str(logging_url), - "headers": headers, - }, - ) - stream = ( - HttpPassThroughEndpointHelpers._update_stream_param_based_on_request_body( - parsed_body=_parsed_body, - stream=stream, - ) - ) - - if stream: - if is_multipart: - response = ( - await HttpPassThroughEndpointHelpers.make_multipart_http_request( - request=request, - async_client=async_client, - url=url, - headers=headers, - requested_query_params=requested_query_params, - stream=True, - ) - ) - else: - # SigV4-signed callers (Bedrock) supply the exact pre-signed bytes; - # otherwise httpx encodes the parsed JSON dict as before. - body_kwargs: Dict[str, Any] = ( - {"content": state_raw_body} - if state_raw_body is not None - else {"json": _parsed_body} - ) - req = async_client.build_request( - request.method, - url, - params=requested_query_params, - headers=headers, - **body_kwargs, - ) - - response = await async_client.send(req, stream=stream) - - try: - response.raise_for_status() - except httpx.HTTPStatusError as e: - raise HTTPException( - status_code=e.response.status_code, detail=await e.response.aread() - ) - - return StreamingResponse( - PassThroughStreamingHandler.chunk_processor( - response=response, - request_body=_parsed_body, - litellm_logging_obj=logging_obj, - endpoint_type=endpoint_type, - start_time=start_time, - passthrough_success_handler_obj=pass_through_endpoint_logging, - url_route=str(url), - ), - headers=HttpPassThroughEndpointHelpers.get_response_headers( - headers=response.headers, - litellm_call_id=litellm_call_id, - ), - status_code=response.status_code, - ) - - if state_raw_body is not None: - # SigV4-signed callers (Bedrock) require the exact pre-signed bytes - # to be forwarded so the signature/Content-Length stay valid. - response = await async_client.request( - method=request.method, - url=url, - headers=headers, - params=requested_query_params, - content=state_raw_body, - ) - else: - response = ( - await HttpPassThroughEndpointHelpers.non_streaming_http_request_handler( - request=request, - async_client=async_client, - url=url, - headers=headers, - requested_query_params=requested_query_params, - _parsed_body=_parsed_body, - forward_multipart=is_multipart, - ) - ) - verbose_proxy_logger.debug("response.headers= %s", response.headers) - - if _is_streaming_response(response) is True: - try: - response.raise_for_status() - except httpx.HTTPStatusError as e: - raise HTTPException( - status_code=e.response.status_code, detail=await e.response.aread() - ) - - return StreamingResponse( - PassThroughStreamingHandler.chunk_processor( - response=response, - request_body=_parsed_body, - litellm_logging_obj=logging_obj, - endpoint_type=endpoint_type, - start_time=start_time, - passthrough_success_handler_obj=pass_through_endpoint_logging, - url_route=str(url), - ), - headers=HttpPassThroughEndpointHelpers.get_response_headers( - headers=response.headers, - litellm_call_id=litellm_call_id, - ), - status_code=response.status_code, - ) - - try: - response.raise_for_status() - except httpx.HTTPStatusError as e: - raise HTTPException( - status_code=e.response.status_code, detail=e.response.text - ) - - if response.status_code >= 300: - raise HTTPException(status_code=response.status_code, detail=response.text) - - content = await response.aread() - - ## POST-CALL GUARDRAILS ## - _content_modified = False - response_body: Optional[dict] = get_response_body(response) - if response_body is not None and guardrails_to_run: - # Build an enriched data dict: _parsed_body has been stripped of - # `metadata` by both pre_call_hook and _init_kwargs_for_pass_through_endpoint, - # so we re-attach the configured guardrails here so should_run_guardrail - # sees them. - hook_data = dict(_parsed_body or {}) - existing_metadata = hook_data.get("metadata") - if not isinstance(existing_metadata, dict): - existing_metadata = {} - hook_data["metadata"] = { - **existing_metadata, - "guardrails": guardrails_to_run, - } - response_body = await proxy_logging_obj.post_call_success_hook( - data=hook_data, - user_api_key_dict=user_api_key_dict, - response=response_body, # type: ignore[arg-type] - ) - if isinstance(response_body, dict): - content = json.dumps(response_body).encode("utf-8") - _content_modified = True - else: - verbose_proxy_logger.debug( - "pass_through_endpoint: post_call_success_hook returned %s, expected dict — using original response", - type(response_body).__name__, - ) - elif response_body is None: - verbose_proxy_logger.debug( - "pass_through_endpoint: response body not JSON-parseable, skipping post-call guardrails" - ) - - ## LOG SUCCESS - passthrough_logging_payload["response_body"] = response_body - end_time = datetime.now() - asyncio.create_task( - pass_through_endpoint_logging.pass_through_async_success_handler( - httpx_response=response, - response_body=response_body, - url_route=str(url), - result="", - start_time=start_time, - end_time=end_time, - logging_obj=logging_obj, - cache_hit=False, - request_body=_parsed_body, - custom_llm_provider=custom_llm_provider, - **kwargs, - ) - ) - - ## CUSTOM HEADERS - `x-litellm-*` - custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( - user_api_key_dict=user_api_key_dict, - call_id=litellm_call_id, - model_id=None, - cache_key=None, - api_base=str(url._uri_reference), - ) - - response_headers = HttpPassThroughEndpointHelpers.get_response_headers( - headers=response.headers, - custom_headers=custom_headers, - ) - if _content_modified: - response_headers.pop("content-length", None) - - return Response( - content=content, - status_code=response.status_code, - headers=response_headers, - ) - except ModifyResponseException as e: - verbose_proxy_logger.info( - "pass_through_endpoint: Guardrail %s modified response: %s", - e.guardrail_name, - str(e.message or "")[:200], - ) - try: - await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, - original_exception=e, - request_data=e.request_data, - ) - except Exception: - verbose_proxy_logger.warning( - "pass_through_endpoint: post_call_failure_hook raised during guardrail block", - exc_info=True, - ) - error_body = { - "error": { - "message": e.message or "Response blocked by guardrail", - "type": "content_filter", - "guardrail_name": e.guardrail_name, - "model": e.model, - } - } - return Response( - content=json.dumps(error_body), - status_code=200, - media_type="application/json", - ) - except Exception as e: - custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( - user_api_key_dict=user_api_key_dict, - call_id=litellm_call_id, - model_id=None, - cache_key=None, - api_base=str(url._uri_reference) if url else None, - ) - verbose_proxy_logger.exception( - "litellm.proxy.proxy_server.pass_through_endpoint(): Exception occured - {}".format( - str(e) - ) - ) - - ######################################################### - # Monitoring: Trigger post_call_failure_hook - # for pass through endpoint failure - ######################################################### - request_payload: dict = _parsed_body or {} - # add user_api_key_dict, litellm_call_id, passthrough_logging_payloa for logging - if kwargs: - for key, value in kwargs.items(): - request_payload[key] = value - if logging_obj is not None: - request_payload["litellm_logging_obj"] = logging_obj - - if ( - "model" not in request_payload - and _parsed_body - and isinstance(_parsed_body, dict) - ): - request_payload["model"] = _parsed_body.get("model", "") - if "custom_llm_provider" not in request_payload and custom_llm_provider: - request_payload["custom_llm_provider"] = custom_llm_provider - - await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, - original_exception=e, - request_data=request_payload, - traceback_str=traceback.format_exc( - limit=MAXIMUM_TRACEBACK_LINES_TO_LOG, - ), - ) - - ######################################################### - - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "message", str(getattr(e, "detail", str(e)))), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - headers=custom_headers, - ) - else: - error_msg = f"{str(e)}" - raise ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", 500), - headers=custom_headers, - ) - - -def _update_metadata_with_tags_in_header(request: Request, metadata: dict) -> dict: - """ - If tags are in the request headers, add them to the metadata - - Used for google and vertex JS SDKs, and Azure passthrough - Checks both 'tags' and 'x-litellm-tags' headers - """ - tags_to_add = [] - - # Check for 'tags' header first - _tags = request.headers.get("tags") - if _tags: - tags_to_add.extend([tag.strip() for tag in _tags.split(",")]) - - _tags = request.headers.get("x-litellm-tags") - if _tags: - tags_to_add.extend([tag.strip() for tag in _tags.split(",")]) - - # Only add tags key if there are tags to add - if tags_to_add: - if "tags" not in metadata: - metadata["tags"] = [] - metadata["tags"].extend(tags_to_add) - - return metadata - - -async def _parse_request_data_by_content_type( - request: Request, -) -> Tuple[Optional[Any], Optional[Any], Optional[Any], Optional[Any]]: - """ - Parse request data based on content type. - - Handles JSON, multipart/form-data, and URL-encoded form data. - - Returns: - Tuple of (query_params_data, custom_body_data, file_data, stream) - """ - content_type = request.headers.get("content-type", "") - - query_params_data = None - custom_body_data = None - file_data = None - stream = None - - if "application/json" in content_type: - # ✅ Handle JSON - try: - body = await request.json() - query_params_data = body.get("query_params") - custom_body_data = body.get("custom_body") - stream = body.get("stream") - except json.JSONDecodeError: - # Handle requests with no body (e.g., DELETE requests) - pass - elif "multipart/form-data" in content_type: - # ✅ Try to parse as JSON first (handles misconfigured clients sending JSON with multipart content-type) - # If that fails, skip parsing - pass_through_request will handle actual multipart - try: - body = await request.json() - # Successfully parsed as JSON - treat as JSON body - query_params_data = body.get("query_params") - custom_body_data = body.get("custom_body") - stream = body.get("stream") - # If custom_body is not set, use the entire body - if custom_body_data is None and body: - custom_body_data = body - except (json.JSONDecodeError, Exception): - # Not JSON - this is actual multipart data - # Skip parsing here to avoid consuming the request body stream - # make_multipart_http_request will handle it - pass - - elif "application/x-www-form-urlencoded" in content_type: - # ✅ Handle URL-encoded form data - form = await request.form() - query_params_data = form.get("query_params") - custom_body_data = form.get("custom_body") - - else: - # ✅ Fallback: maybe no body, just query params - query_params_data = dict(request.query_params) or None - - return query_params_data, custom_body_data, file_data, stream - - -def create_pass_through_route( - endpoint, - target: str, - custom_headers: Optional[Mapping[str, Any]] = None, - _forward_headers: Optional[bool] = False, - _merge_query_params: Optional[bool] = False, - dependencies: Optional[List] = None, - include_subpath: Optional[bool] = False, - cost_per_request: Optional[float] = None, - custom_llm_provider: Optional[str] = None, - is_streaming_request: Optional[bool] = False, - query_params: Optional[dict] = None, - default_query_params: Optional[dict] = None, - guardrails: Optional[Dict[str, Any]] = None, - config_file_path: Optional[str] = None, -): - # check if target is an adapter.py or a url - from litellm._uuid import uuid - from litellm.proxy.types_utils.utils import get_instance_fn - - try: - if isinstance(target, CustomLogger): - adapter = target - else: - adapter = get_instance_fn(value=target, config_file_path=config_file_path) - adapter_id = str(uuid.uuid4()) - litellm.adapters = [{"id": adapter_id, "adapter": adapter}] - - async def endpoint_func( # type: ignore - request: Request, - fastapi_response: Response, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), - subpath: str = "", # captures sub-paths when include_subpath=True - ): - return await chat_completion_pass_through_endpoint( - fastapi_response=fastapi_response, - request=request, - adapter_id=adapter_id, - user_api_key_dict=user_api_key_dict, - ) - - except Exception: - verbose_proxy_logger.debug("Defaulting to target being a url.") - - async def endpoint_func( # type: ignore - request: Request, - fastapi_response: Response, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), - subpath: str = "", # captures sub-paths when include_subpath=True - ): - from litellm.proxy.auth.auth_utils import ( # noqa: PLC0415 - get_request_route, - ) - from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( - InitPassThroughEndpointHelpers, - ) - - path = get_request_route(request) - - # Parse request data based on content type - ( - query_params_data, - custom_body_data, - file_data, - stream, - ) = await _parse_request_data_by_content_type(request) - - if not InitPassThroughEndpointHelpers.is_registered_pass_through_route( - route=path - ): - raise HTTPException( - status_code=404, - detail=f"Pass-through endpoint {endpoint} not found. This could have been deleted or not yet added to the proxy.", - ) - - passthrough_params = ( - InitPassThroughEndpointHelpers.get_registered_pass_through_route( - route=path, method=request.method - ) - ) - target_params = { - "target": target, - "custom_headers": custom_headers, - "forward_headers": _forward_headers, - "merge_query_params": _merge_query_params, - "cost_per_request": cost_per_request, - "guardrails": None, - } - - if passthrough_params is not None: - target_params.update(passthrough_params.get("passthrough_params", {})) - - # Extract and cast parameters with proper types - param_target = target_params.get("target") or target - param_custom_headers = target_params.get("custom_headers", custom_headers) - param_forward_headers = target_params.get( - "forward_headers", _forward_headers - ) - param_merge_query_params = target_params.get( - "merge_query_params", _merge_query_params - ) - param_cost_per_request = target_params.get( - "cost_per_request", cost_per_request - ) - param_guardrails = target_params.get("guardrails", None) - param_default_query_params = target_params.get("default_query_params", None) - - # Construct the full target URL with subpath if needed - full_target = ( - HttpPassThroughEndpointHelpers.construct_target_url_with_subpath( - base_target=cast(str, param_target), - subpath=subpath, - include_subpath=include_subpath, - ) - ) - - # Ensure custom_headers is a dict. Botocore returns a HeadersDict - # for SigV4-prepared requests, which is a Mapping but not a dict. - headers_dict = ( - dict(param_custom_headers) - if isinstance(param_custom_headers, Mapping) - else {} - ) - - # Ensure query_params and custom_body are dicts or None - final_query_params = ( - query_params_data if isinstance(query_params_data, dict) else {} - ) - if query_params: - final_query_params.update(query_params) - # Programmatic callers set LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY on - # request.state (see Bedrock proxy). Parsed JSON envelope otherwise. - state_custom_body: Optional[dict] = getattr( - request.state, - LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, - None, - ) - final_custom_body: Optional[dict] = None - if isinstance(state_custom_body, dict): - final_custom_body = state_custom_body - elif isinstance(custom_body_data, dict): - final_custom_body = custom_body_data - - try: - return await pass_through_request( # type: ignore - request=request, - target=full_target, - custom_headers=headers_dict, - user_api_key_dict=user_api_key_dict, - forward_headers=cast(Optional[bool], param_forward_headers), - merge_query_params=cast(Optional[bool], param_merge_query_params), - query_params=final_query_params, - default_query_params=cast( - Optional[dict], param_default_query_params - ), - stream=is_streaming_request or stream, - custom_body=final_custom_body, - cost_per_request=cast(Optional[float], param_cost_per_request), - custom_llm_provider=custom_llm_provider, - guardrails_config=cast(Optional[dict], param_guardrails), - ) - finally: - if hasattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY): - delattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY) - if hasattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY): - delattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY) - - return endpoint_func - - -def create_websocket_passthrough_route( - endpoint: str, - target: str, - custom_headers: Optional[dict] = None, - _forward_headers: Optional[bool] = False, - dependencies: Optional[List] = None, - cost_per_request: Optional[float] = None, -): - """ - Create a WebSocket passthrough route function. - - Args: - endpoint: The endpoint path (for logging purposes) - target: The target WebSocket URL (e.g., "wss://api.example.com/ws") - custom_headers: Custom headers to include in the WebSocket connection - _forward_headers: Whether to forward incoming headers - dependencies: FastAPI dependencies to inject - - Returns: - A WebSocket passthrough function that can be registered with app.websocket() - """ - from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket - - async def websocket_endpoint_func( - websocket: WebSocket, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket), - **kwargs, # For additional query parameters - ): - """ - WebSocket passthrough endpoint function. - - This function handles the WebSocket connection by: - 1. Accepting the incoming WebSocket connection - 2. Establishing a connection to the target WebSocket - 3. Forwarding messages bidirectionally - 4. Handling connection cleanup - """ - return await websocket_passthrough_request( - websocket=websocket, - target=target, - custom_headers=custom_headers or {}, - user_api_key_dict=user_api_key_dict, - forward_headers=_forward_headers, - endpoint=endpoint, - cost_per_request=cost_per_request, - accept_websocket=True, # Generic usage should accept the WebSocket - ) - - return websocket_endpoint_func - - -async def websocket_passthrough_request( # noqa: PLR0915 - websocket: WebSocket, - target: str, - custom_headers: dict, - user_api_key_dict: UserAPIKeyAuth, - forward_headers: Optional[bool] = False, - endpoint: Optional[str] = None, - cost_per_request: Optional[float] = None, - accept_websocket: bool = True, -): - """ - WebSocket passthrough request handler. - - Args: - websocket: The incoming WebSocket connection - target: The target WebSocket URL - custom_headers: Custom headers to include in the connection - user_api_key_dict: The user API key dictionary - forward_headers: Whether to forward incoming headers - endpoint: The endpoint path (for logging purposes) - cost_per_request: Optional field - cost per request to the target endpoint - """ - from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.proxy.proxy_server import proxy_logging_obj - from litellm.types.passthrough_endpoints.pass_through_endpoints import ( - PassthroughStandardLoggingPayload, - ) - - # Initialize tracking variables - start_time = datetime.now() - websocket_messages: list[dict[str, Any]] = [] - litellm_call_id = str(uuid.uuid4()) - - verbose_proxy_logger.info( - f"WebSocket passthrough ({endpoint}): Starting WebSocket connection to {target}" - ) - - # Only accept the WebSocket if requested (for generic usage) - if accept_websocket: - await websocket.accept() - verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): WebSocket connection accepted" - ) - - # Prepare headers for the upstream connection - upstream_headers = custom_headers.copy() - - if forward_headers: - # Forward relevant headers from the incoming request - incoming_headers = dict(websocket.headers) - for header_name, header_value in incoming_headers.items(): - # Only forward certain headers to avoid conflicts - if header_name.lower() in [ - "authorization", - "x-api-key", - "x-goog-user-project", - ]: - upstream_headers[header_name] = header_value - - # Initialize logging object similar to HTTP passthrough - logging_obj = Logging( - model="unknown", - messages=[{"role": "user", "content": "WebSocket connection"}], - stream=True, # WebSockets are inherently streaming - call_type="pass_through_endpoint", - start_time=start_time, - litellm_call_id=litellm_call_id, - function_id="websocket_passthrough", - ) - - # Create passthrough logging payload - passthrough_logging_payload = PassthroughStandardLoggingPayload( - url=target, - request_body={}, # WebSocket doesn't have a traditional request body - request_method="WEBSOCKET", - cost_per_request=cost_per_request, - ) - - # Create a dummy request object for WebSocket connections to maintain compatibility - # with the existing _init_kwargs_for_pass_through_endpoint function - class DummyRequest: - def __init__( - self, url: str, method: str = "WEBSOCKET", headers: Optional[dict] = None - ): - self.url = url - self.method = method - self.headers = headers or {} - - def __str__(self): - return f"DummyRequest(url={self.url}, method={self.method})" - - dummy_request = DummyRequest( - url=target, - method="WEBSOCKET", - headers=dict(websocket.headers) if hasattr(websocket, "headers") else {}, - ) - - # Initialize kwargs for logging using the same pattern as HTTP passthrough - kwargs = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( - user_api_key_dict=user_api_key_dict, - _parsed_body={}, # WebSocket doesn't have a traditional request body - passthrough_logging_payload=passthrough_logging_payload, - litellm_call_id=litellm_call_id, - request=dummy_request, # type: ignore - logging_obj=logging_obj, - ) - - # Update logging environment variables - logging_obj.update_environment_variables( - model="unknown", - user="unknown", - optional_params={}, - litellm_params=dict(kwargs.get("litellm_params", {})), - call_type="pass_through_endpoint", - ) - logging_obj.model_call_details["litellm_call_id"] = litellm_call_id - - # Pre-call logging - logging_obj.pre_call( - input=[{"role": "user", "content": "WebSocket connection"}], - api_key="", - additional_args={ - "complete_input_dict": {}, - "api_base": target, - "headers": upstream_headers, - }, - ) - - ### CALL HOOKS ### - modify incoming data / reject request before calling the model - websocket_data: dict[str, Any] = {} - websocket_data = await proxy_logging_obj.pre_call_hook( - user_api_key_dict=user_api_key_dict, - data=websocket_data, - call_type="pass_through_endpoint", - ) - - try: - verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Establishing upstream connection to {target}" - ) - async with connect( - target, - additional_headers=upstream_headers, - ) as upstream_ws: - verbose_proxy_logger.info( - f"WebSocket passthrough ({endpoint}): Upstream connection established successfully" - ) - - async def forward_client_to_upstream() -> None: - """Forward messages from client to upstream WebSocket""" - try: - while True: - message = await websocket.receive() - message_type = message.get("type") - if message_type == "websocket.disconnect": - await upstream_ws.close() - break - - text_data = message.get("text") - bytes_data = message.get("bytes") - - if text_data is not None: - # Try to extract model from client setup message for Vertex AI Live - if endpoint and "/vertex_ai/live" in endpoint: - verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Processing client message for model extraction" - ) - try: - client_message = json.loads(text_data) - if ( - isinstance(client_message, dict) - and "setup" in client_message - ): - setup_data = client_message["setup"] - verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Found setup data in client message: {setup_data}" - ) - if ( - isinstance(setup_data, dict) - and "model" in setup_data - ): - extracted_model = ( - _extract_model_from_vertex_ai_setup( - setup_data - ) - ) - if extracted_model: - kwargs["model"] = extracted_model - kwargs["custom_llm_provider"] = ( - "vertex_ai-language-models" - ) - # Update logging object with correct model - logging_obj.model = extracted_model - logging_obj.model_call_details[ - "model" - ] = extracted_model - logging_obj.model_call_details[ - "custom_llm_provider" - ] = "vertex_ai" - verbose_proxy_logger.info( - f"WebSocket passthrough ({endpoint}): Successfully extracted model '{extracted_model}' and set provider to 'vertex_ai' from client setup message" - ) - else: - verbose_proxy_logger.warning( - f"WebSocket passthrough ({endpoint}): Failed to extract model from client setup data: {setup_data}" - ) - else: - verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Setup data does not contain model field: {setup_data}" - ) - else: - verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Client message does not contain setup data" - ) - except (json.JSONDecodeError, KeyError, TypeError) as e: - verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Client message is not a valid setup message: {e}" - ) - pass # Not a JSON message or doesn't contain setup data - - await upstream_ws.send(text_data) - elif bytes_data is not None: - await upstream_ws.send(bytes_data) - except asyncio.CancelledError: - raise - except Exception: - verbose_proxy_logger.exception( - f"WebSocket passthrough ({endpoint}): error forwarding client message" - ) - await upstream_ws.close() - - async def forward_upstream_to_client() -> None: - """Forward messages from upstream to client WebSocket""" - try: - # Wait for the first response from upstream - raw_response = await upstream_ws.recv(decode=False) - # Ensure raw_response is bytes before decoding - if isinstance(raw_response, str): - raw_response = raw_response.encode("ascii") - setup_response = json.loads(raw_response.decode("ascii")) - verbose_proxy_logger.debug(f"Setup response: {setup_response}") - - # Extract model and provider from setup response for Vertex AI Live - if endpoint and "/vertex_ai/live" in endpoint: - verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Processing server setup response for model extraction" - ) - extracted_model = _extract_model_from_vertex_ai_setup( - setup_response - ) - if extracted_model: - kwargs["model"] = extracted_model - kwargs["custom_llm_provider"] = "vertex_ai_language_models" - # Update logging object with correct model - logging_obj.model = extracted_model - logging_obj.model_call_details["model"] = extracted_model - logging_obj.model_call_details["custom_llm_provider"] = ( - "vertex_ai_language_models" - ) - verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Successfully extracted model '{extracted_model}' and set provider to 'vertex_ai' from server setup response" - ) - else: - verbose_proxy_logger.warning( - f"WebSocket passthrough ({endpoint}): Failed to extract model from server setup response: {setup_response}" - ) - else: - verbose_proxy_logger.debug( - f"WebSocket passthrough ({endpoint}): Not a Vertex AI Live endpoint, skipping model extraction" - ) - - # Send the setup response to the client - await websocket.send_text(json.dumps(setup_response)) - - # Now continuously forward messages from upstream to client - async for upstream_message in upstream_ws: - if isinstance(upstream_message, bytes): - await websocket.send_bytes(upstream_message) - # Parse and collect for cost tracking - try: - message_data = json.loads(upstream_message.decode()) - websocket_messages.append(message_data) - except (json.JSONDecodeError, UnicodeDecodeError): - pass - else: - await websocket.send_text(upstream_message) - # Parse and collect for cost tracking - try: - message_data = json.loads(upstream_message) - websocket_messages.append(message_data) - except json.JSONDecodeError: - pass - - except (ConnectionClosedOK, ConnectionClosedError) as e: - verbose_proxy_logger.debug( - f"Upstream WebSocket connection closed: {e}" - ) - pass - except asyncio.CancelledError: - verbose_proxy_logger.debug( - "asyncio.CancelledError in forward_upstream_to_client" - ) - raise - except Exception as e: - verbose_proxy_logger.debug( - f"Exception in forward_upstream_to_client: {e}" - ) - verbose_proxy_logger.exception( - f"WebSocket passthrough ({endpoint}): error forwarding upstream message" - ) - raise - - # Create tasks for bidirectional message forwarding - tasks = [ - asyncio.create_task(forward_client_to_upstream()), - asyncio.create_task(forward_upstream_to_client()), - ] - - done, pending = await asyncio.wait( - tasks, return_when=asyncio.FIRST_COMPLETED - ) - - # Cancel remaining tasks - for task in pending: - task.cancel() - try: - await task - except asyncio.CancelledError: - pass - - # Check for exceptions in completed tasks - for task in done: - exception = task.exception() - if exception is not None: - raise exception - - end_time = datetime.now() - - # Update passthrough logging payload with response data - passthrough_logging_payload["response_body"] = websocket_messages # type: ignore - passthrough_logging_payload["end_time"] = end_time # type: ignore - - # Remove logging_obj from kwargs to avoid duplicate keyword argument - success_kwargs = kwargs.copy() - success_kwargs.pop("logging_obj", None) - - # # Add user authentication context for database logging - # if user_api_key_dict: - # success_kwargs.setdefault('litellm_params', {}) - # success_kwargs['litellm_params'].update({ - # 'proxy_server_request': { - # 'body': { - # 'user': user_api_key_dict.user_id, - # 'team_id': user_api_key_dict.team_id, - # 'end_user_id': user_api_key_dict.end_user_id, - # } - # } - # }) - # # Also add the user_api_key for direct access - # success_kwargs['user_api_key'] = user_api_key_dict.api_key - - # Create a dummy httpx.Response for WebSocket connections - class MockWebSocketResponse: - def __init__(self, target_url: str): - self.status_code = 200 - self.text = "WebSocket connection successful" - self.headers: dict[str, str] = {} - self.request = MockWebSocketRequest(target_url) - - class MockWebSocketRequest: - def __init__(self, target_url: str): - self.method = "WEBSOCKET" - self.url = target_url - - mock_response = MockWebSocketResponse(target) - - # Use the same success handler as HTTP passthrough endpoints - asyncio.create_task( - pass_through_endpoint_logging.pass_through_async_success_handler( - httpx_response=mock_response, # type: ignore - response_body=websocket_messages, # type: ignore - url_route=endpoint or "", - result="websocket_connection_successful", - start_time=start_time, - end_time=end_time, - logging_obj=logging_obj, - cache_hit=False, - request_body={}, - **success_kwargs, - ) - ) - - # Call the proxy logging success hook - if proxy_logging_obj: - await proxy_logging_obj.post_call_success_hook( - data={}, - user_api_key_dict=user_api_key_dict, - response={"status": "websocket_connection_successful"}, # type: ignore - ) - - except InvalidStatus as exc: - verbose_proxy_logger.exception( - f"WebSocket passthrough ({endpoint}): upstream rejected WebSocket connection" - ) - - # Prepare request payload for logging - request_payload = {} - if kwargs: - for key, value in kwargs.items(): - request_payload[key] = value - if logging_obj is not None: - request_payload["litellm_logging_obj"] = logging_obj - - # Log the connection failure using the same pattern as HTTP - await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, - original_exception=exc, - request_data=request_payload, - traceback_str=traceback.format_exc( - limit=MAXIMUM_TRACEBACK_LINES_TO_LOG, - ), - ) - - if websocket.client_state != WebSocketState.DISCONNECTED: - await websocket.close( - code=getattr(exc, "status_code", 1011), - reason="Upstream connection rejected", - ) - except Exception as e: - verbose_proxy_logger.exception( - f"WebSocket passthrough ({endpoint}): unexpected error while proxying WebSocket" - ) - - # Prepare request payload for logging - request_payload = {} - if kwargs: - for key, value in kwargs.items(): - request_payload[key] = value - if logging_obj is not None: - request_payload["litellm_logging_obj"] = logging_obj - - # Log the unexpected error using the same pattern as HTTP - await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, - original_exception=e, - request_data=request_payload, - traceback_str=traceback.format_exc( - limit=MAXIMUM_TRACEBACK_LINES_TO_LOG, - ), - ) - - if websocket.client_state != WebSocketState.DISCONNECTED: - await websocket.close(code=1011, reason="WebSocket passthrough error") - finally: - if websocket.client_state != WebSocketState.DISCONNECTED: - await websocket.close() - - -def _is_streaming_response(response: httpx.Response) -> bool: - _content_type = response.headers.get("content-type") - if _content_type is not None and "text/event-stream" in _content_type: - return True - return False - - -def _extract_model_from_vertex_ai_setup(setup_response: dict) -> Optional[str]: - """ - Extract the model name from Vertex AI Live setup response. - - The setup response can contain a model field in two formats: - 1. Direct: {"model": "projects/.../models/gemini-2.0-flash-live-preview-04-09"} - 2. Nested: {"setup": {"model": "projects/.../models/gemini-2.0-flash-live-preview-04-09"}} - - We extract just the model name: "gemini-2.0-flash-live-preview-04-09" - """ - try: - # Handle both direct model field and nested setup.model field - model_path = None - if isinstance(setup_response, dict): - if "model" in setup_response: - model_path = setup_response["model"] - elif ( - "setup" in setup_response - and isinstance(setup_response["setup"], dict) - and "model" in setup_response["setup"] - ): - model_path = setup_response["setup"]["model"] - - if isinstance(model_path, str) and "/models/" in model_path: - # Extract the model name after the last "/models/" - model_name = model_path.split("/models/")[-1] - return model_name - except Exception as e: - verbose_proxy_logger.debug(f"Error extracting model from setup response: {e}") - return None - - -class SafeRouteAdder: - """ - Wrapper class for adding routes to FastAPI app. - Only adds routes if they don't already exist on the app. - """ - - @staticmethod - def _is_path_registered(app: FastAPI, path: str, methods: List[str]) -> bool: - """ - Check if a path with any of the specified methods is already registered on the app. - - Args: - app: The FastAPI application instance - path: The path to check (e.g., "/v1/chat/completions") - methods: List of HTTP methods to check (e.g., ["GET", "POST"]) - - Returns: - True if the path is already registered with any of the methods, False otherwise - """ - for route in app.routes: - # Use getattr to safely access route attributes - route_path = getattr(route, "path", None) - route_methods = getattr(route, "methods", None) - - if route_path == path and route_methods is not None: - # Check if any of the methods overlap - if any(method in route_methods for method in methods): - return True - return False - - @staticmethod - def add_api_route_if_not_exists( - app: FastAPI, - path: str, - endpoint: Any, - methods: List[str], - dependencies: Optional[List] = None, - ) -> bool: - """ - Add an API route to the app only if it doesn't already exist. - - Args: - app: The FastAPI application instance - path: The path for the route - endpoint: The endpoint function/callable - methods: List of HTTP methods - dependencies: Optional list of dependencies - - Returns: - True if route was added, False if it already existed - """ - if SafeRouteAdder._is_path_registered(app=app, path=path, methods=methods): - verbose_proxy_logger.debug( - "Skipping route registration - path %s with methods %s already registered on app", - path, - methods, - ) - return False - - app.add_api_route( - path=path, - endpoint=endpoint, - methods=methods, - dependencies=dependencies, - ) - verbose_proxy_logger.debug( - "Successfully added route: %s with methods %s", - path, - methods, - ) - return True - - -class InitPassThroughEndpointHelpers: - @staticmethod - def add_exact_path_route( - app: FastAPI, - path: str, - target: str, - custom_headers: Optional[dict], - forward_headers: Optional[bool], - merge_query_params: Optional[bool], - dependencies: Optional[List], - cost_per_request: Optional[float], - endpoint_id: str, - guardrails: Optional[dict] = None, - methods: Optional[List[str]] = None, - default_query_params: Optional[dict] = None, - config_file_path: Optional[str] = None, - ): - """Add exact path route for pass-through endpoint""" - # Default to all methods if none specified (backward compatibility) - if methods is None or len(methods) == 0: - methods = ["GET", "POST", "PUT", "DELETE", "PATCH"] - - # Create route key that includes methods for uniqueness - methods_str = ",".join(sorted(methods)) - route_key = f"{endpoint_id}:exact:{path}:{methods_str}" - - # Check if this exact route is already registered - if route_key in _registered_pass_through_routes: - verbose_proxy_logger.debug( - "Updating duplicate exact pass through endpoint: %s with methods %s (already registered)", - path, - methods, - ) - - verbose_proxy_logger.debug( - "adding exact pass through endpoint: %s, methods: %s, dependencies: %s", - path, - methods, - dependencies, - ) - - # Use SafeRouteAdder to only add route if it doesn't exist on the app - SafeRouteAdder.add_api_route_if_not_exists( - app=app, - path=path, - endpoint=create_pass_through_route( # type: ignore - path, - target, - custom_headers, - forward_headers, - merge_query_params, - dependencies, - cost_per_request=cost_per_request, - default_query_params=default_query_params, - guardrails=guardrails, - config_file_path=config_file_path, - ), - methods=methods, - dependencies=dependencies, - ) - - # Always register/update the route metadata (headers, target) even if FastAPI route exists - _registered_pass_through_routes[route_key] = { - "endpoint_id": endpoint_id, - "path": path, - "type": "exact", - "methods": methods, - "passthrough_params": { - "target": target, - "custom_headers": custom_headers, - "forward_headers": forward_headers, - "merge_query_params": merge_query_params, - "default_query_params": default_query_params, - "dependencies": dependencies, - "cost_per_request": cost_per_request, - "guardrails": guardrails, - }, - } - - @staticmethod - def add_subpath_route( - app: FastAPI, - path: str, - target: str, - custom_headers: Optional[dict], - forward_headers: Optional[bool], - merge_query_params: Optional[bool], - dependencies: Optional[List], - cost_per_request: Optional[float], - endpoint_id: str, - guardrails: Optional[dict] = None, - methods: Optional[List[str]] = None, - default_query_params: Optional[dict] = None, - config_file_path: Optional[str] = None, - ): - """Add wildcard route for sub-paths""" - # Default to all methods if none specified (backward compatibility) - if methods is None or len(methods) == 0: - methods = ["GET", "POST", "PUT", "DELETE", "PATCH"] - - wildcard_path = f"{path}/{{subpath:path}}" - methods_str = ",".join(sorted(methods)) - route_key = f"{endpoint_id}:subpath:{path}:{methods_str}" - - # Check if this subpath route is already registered - if route_key in _registered_pass_through_routes: - verbose_proxy_logger.debug( - "Updating duplicate wildcard pass through endpoint: %s with methods %s (already registered)", - wildcard_path, - methods, - ) - - verbose_proxy_logger.debug( - "adding wildcard pass through endpoint: %s, methods: %s, dependencies: %s", - wildcard_path, - methods, - dependencies, - ) - - # Use SafeRouteAdder to only add route if it doesn't exist on the app - SafeRouteAdder.add_api_route_if_not_exists( - app=app, - path=wildcard_path, - endpoint=create_pass_through_route( # type: ignore - path, - target, - custom_headers, - forward_headers, - merge_query_params, - dependencies, - include_subpath=True, - cost_per_request=cost_per_request, - default_query_params=default_query_params, - guardrails=guardrails, - config_file_path=config_file_path, - ), - methods=methods, - dependencies=dependencies, - ) - - # Register the route to prevent duplicates only if it was added - _registered_pass_through_routes[route_key] = { - "endpoint_id": endpoint_id, - "path": path, - "type": "subpath", - "methods": methods, - "passthrough_params": { - "target": target, - "custom_headers": custom_headers, - "forward_headers": forward_headers, - "merge_query_params": merge_query_params, - "default_query_params": default_query_params, - "dependencies": dependencies, - "cost_per_request": cost_per_request, - "guardrails": guardrails, - }, - } - - @staticmethod - def remove_endpoint_routes(endpoint_id: str): - """Remove all routes for a specific endpoint ID from the registry - and clean up corresponding entries from LiteLLMRoutes.openai_routes.""" - keys_to_remove = [ - key - for key, value in _registered_pass_through_routes.items() - if value["endpoint_id"] == endpoint_id - ] - for key in keys_to_remove: - route_info = _registered_pass_through_routes[key] - path = route_info.get("path") - if isinstance(path, str): - openai_routes = LiteLLMRoutes.openai_routes.value - if path in openai_routes: - openai_routes.remove(path) - if route_info.get("type") == "subpath": - wildcard_path = path.rstrip("/") + "/*" - if wildcard_path in openai_routes: - openai_routes.remove(wildcard_path) - del _registered_pass_through_routes[key] - verbose_proxy_logger.debug( - "Removed pass-through route from registry: %s", key - ) - - @staticmethod - def clear_all_pass_through_routes(): - """Clear all pass-through routes from the registry""" - _registered_pass_through_routes.clear() - - @staticmethod - def get_all_registered_pass_through_routes() -> List[str]: - """Get all registered pass-through endpoints from the registry""" - return list(_registered_pass_through_routes.keys()) - - @staticmethod - def _build_full_path_with_root(path: str) -> str: - """ - Build full path by prepending server root path if needed. - - Args: - path: The relative path to build - - Returns: - Full path with server root prepended (if root is not "/") - """ - root_path = get_server_root_path() - if root_path == "/": - return path - return f"{root_path}{path}" - - @staticmethod - def is_registered_pass_through_route(route: str) -> bool: - """ - Check if route is a registered pass-through endpoint from DB - - Uses the in-memory registry to avoid additional DB queries - Optimized for minimal latency - - Args: - route: The route to check - - Returns: - bool: True if route is a registered pass-through endpoint, False otherwise - """ - ## CHECK IF MAPPED PASS THROUGH ENDPOINT - normalized_route = normalize_route_for_root_path(route) - if normalized_route is not None: - for mapped_route in LiteLLMRoutes.mapped_pass_through_routes.value: - if normalized_route.startswith(mapped_route): - return True - - # Fast path: check if any registered route key contains this path - # Keys are in format: "{endpoint_id}:exact:{path}:{methods}" or "{endpoint_id}:subpath:{path}:{methods}" - # For backward compatibility, also support old format: "{endpoint_id}:exact:{path}" or "{endpoint_id}:subpath:{path}" - # Extract unique paths from keys for quick checking - for key in _registered_pass_through_routes.keys(): - parts = key.split(":", 3) # Split into [endpoint_id, type, path, methods?] - if len(parts) >= 3: - route_type = parts[1] - registered_path = ( - InitPassThroughEndpointHelpers._build_full_path_with_root(parts[2]) - ) - if route_type == "exact" and route == registered_path: - return True - elif route_type == "subpath": - if route == registered_path or route.startswith( - registered_path + "/" - ): - return True - - return False - - @staticmethod - def get_registered_pass_through_route( - route: str, method: Optional[str] = None - ) -> Optional[Dict[str, Any]]: - """Get passthrough params for a given route and optionally filter by HTTP method""" - for key in _registered_pass_through_routes.keys(): - parts = key.split(":", 3) # Split into [endpoint_id, type, path, methods?] - if len(parts) >= 3: - route_type = parts[1] - registered_path = ( - InitPassThroughEndpointHelpers._build_full_path_with_root(parts[2]) - ) - - # Get the methods for this route - route_methods = _registered_pass_through_routes[key].get("methods", []) - - # Check if path matches - path_matches = False - if route_type == "exact" and route == registered_path: - path_matches = True - elif route_type == "subpath": - if route == registered_path or route.startswith( - registered_path + "/" - ): - path_matches = True - - # If path matches and method filter is provided, check if method is allowed - if path_matches: - if method is None or not route_methods or method in route_methods: - return _registered_pass_through_routes[key] - - return None - - -def _get_combined_pass_through_endpoints( - pass_through_endpoints: Union[List[Dict], List[PassThroughGenericEndpoint]], - config_pass_through_endpoints: List[Dict], -): - """Get combined pass-through endpoints from db + config""" - return pass_through_endpoints + config_pass_through_endpoints - - -async def _register_pass_through_endpoint( - endpoint: Union[Dict[str, Any], PassThroughGenericEndpoint], - app: FastAPI, - premium_user: bool, - visited_endpoints: set[str], - config_file_path: Optional[str] = None, -) -> None: - endpoint_data: Dict[str, Any] - if isinstance(endpoint, PassThroughGenericEndpoint): - endpoint_data = endpoint.model_dump() - else: - endpoint_data = endpoint - - if endpoint_data.get("id") is None: - endpoint_data["id"] = str(uuid.uuid4()) - endpoint_id = cast(str, endpoint_data["id"]) - - target = endpoint_data.get("target") - path = endpoint_data.get("path") - if path is None: - raise ValueError("Path is required for pass-through endpoint") - - custom_headers = await set_env_variables_in_header( - custom_headers=endpoint_data.get("headers") - ) - forward_headers = endpoint_data.get("forward_headers") - merge_query_params = endpoint_data.get("merge_query_params") - default_query_params = endpoint_data.get("default_query_params") - auth = endpoint_data.get("auth") - dependencies = None - - if auth is not None and str(auth).lower() == "true": - # Authentication on a pass-through endpoint used to be enterprise-only. - # That left OSS with no safe configuration: auth=True raised at startup - # unless the operator had a license. The safe option must always be free, - # and unauthenticated forwarding should require explicit opt-in. - dependencies = [Depends(user_api_key_auth)] - if path not in LiteLLMRoutes.openai_routes.value: - LiteLLMRoutes.openai_routes.value.append(path) - - if target is None: - return - - guardrails = endpoint_data.get("guardrails") - methods = endpoint_data.get("methods") - cost_per_request = endpoint_data.get("cost_per_request") - - verbose_proxy_logger.debug( - "Initializing pass through endpoint: %s (ID: %s)", path, endpoint_id - ) - InitPassThroughEndpointHelpers.add_exact_path_route( - app=app, - path=path, - target=target, - custom_headers=custom_headers, - forward_headers=forward_headers, - merge_query_params=merge_query_params, - dependencies=dependencies, - cost_per_request=cost_per_request, - endpoint_id=endpoint_id, - guardrails=guardrails, - methods=methods, - default_query_params=default_query_params, - config_file_path=config_file_path, - ) - - methods_for_key = methods if methods else ["GET", "POST", "PUT", "DELETE", "PATCH"] - methods_str = ",".join(sorted(methods_for_key)) - visited_endpoints.add(f"{endpoint_id}:exact:{path}:{methods_str}") - - if endpoint_data.get("include_subpath", False) is True: - if auth is not None and str(auth).lower() == "true": - wildcard_path = path.rstrip("/") + "/*" - if wildcard_path not in LiteLLMRoutes.openai_routes.value: - LiteLLMRoutes.openai_routes.value.append(wildcard_path) - InitPassThroughEndpointHelpers.add_subpath_route( - app=app, - path=path, - target=target, - custom_headers=custom_headers, - forward_headers=forward_headers, - merge_query_params=merge_query_params, - dependencies=dependencies, - cost_per_request=cost_per_request, - endpoint_id=endpoint_id, - guardrails=guardrails, - methods=methods, - default_query_params=default_query_params, - config_file_path=config_file_path, - ) - visited_endpoints.add(f"{endpoint_id}:subpath:{path}:{methods_str}") - - verbose_proxy_logger.debug( - "Added new pass through endpoint: %s (ID: %s)", path, endpoint_id - ) - - -async def initialize_pass_through_endpoints( - pass_through_endpoints: Union[List[Dict], List[PassThroughGenericEndpoint]], - config_file_path: Optional[str] = None, -): - """ - 1. Create a global list of pass-through endpoints (db + config) - 2. Clear all existing pass-through endpoints from the FastAPI app routes - 3. Add new endpoints to the in-memory registry - - Initialize a list of pass-through endpoints by adding them to the FastAPI app routes - - Args: - pass_through_endpoints: List of pass-through endpoints to initialize - config_file_path: Path to the operator's config.yaml when this call - originates from a YAML-load. Threaded through to - ``create_pass_through_route`` so an operator using - ``s3://``/``gcs://`` ``custom_handler`` in their config still - loads. Callers from the DB-overlay / runtime API path must leave - this ``None`` so the runtime gate in ``get_instance_fn`` fires. - - Returns: - None - """ - verbose_proxy_logger.debug("initializing pass through endpoints") - from litellm.proxy.proxy_server import ( - app, - config_passthrough_endpoints, - premium_user, - ) - - ## get combined pass-through endpoints from db + config - combined_pass_through_endpoints: List[Union[Dict, PassThroughGenericEndpoint]] - - if config_passthrough_endpoints is not None: - combined_pass_through_endpoints = _get_combined_pass_through_endpoints( # type: ignore - pass_through_endpoints, config_passthrough_endpoints - ) - else: - combined_pass_through_endpoints = pass_through_endpoints # type: ignore - - ## clear all existing pass-through endpoints from the FastAPI app routes - # InitPassThroughEndpointHelpers.clear_all_pass_through_routes() - - # get a list of all registered pass-through endpoints - # mark the ones that are visited in the list - # remove the ones that are not visited from the list - registered_pass_through_endpoints = ( - InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() - ) - - visited_endpoints: set[str] = set() - - for endpoint in combined_pass_through_endpoints: - await _register_pass_through_endpoint( - endpoint=endpoint, - app=app, - premium_user=premium_user, - visited_endpoints=visited_endpoints, - config_file_path=config_file_path, - ) - - # remove the ones that are not visited from the list - for endpoint_key in registered_pass_through_endpoints: - if endpoint_key not in visited_endpoints: - InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_key) - - -def _get_pass_through_endpoints_from_config() -> List[PassThroughGenericEndpoint]: - """ - Get pass-through endpoints defined in the config file. - These are read-only and cannot be edited via the UI. - Malformed endpoints are logged and skipped; they do not crash the function. - """ - from pydantic import ValidationError - - from litellm.proxy.proxy_server import config_passthrough_endpoints - - if config_passthrough_endpoints is None or len(config_passthrough_endpoints) == 0: - return [] - - returned_endpoints: List[PassThroughGenericEndpoint] = [] - for endpoint in config_passthrough_endpoints: - try: - if isinstance(endpoint, dict): - endpoint_dict = dict(endpoint) - endpoint_dict["is_from_config"] = True - returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) - elif isinstance(endpoint, PassThroughGenericEndpoint): - # Create a copy with is_from_config=True - endpoint_dict = endpoint.model_dump() - endpoint_dict["is_from_config"] = True - returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) - except ValidationError as e: - verbose_proxy_logger.warning( - "Skipping malformed pass-through endpoint from config: %s", - e, - exc_info=False, - ) - - return returned_endpoints - - -async def _get_pass_through_endpoints_from_db( - endpoint_id: Optional[str] = None, - user_api_key_dict: Optional[UserAPIKeyAuth] = None, -) -> List[PassThroughGenericEndpoint]: - from litellm.proxy._types import LitellmUserRoles - from litellm.proxy.proxy_server import get_config_general_settings - - try: - if user_api_key_dict is None: - user_api_key_dict = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) - response: ConfigFieldInfo = await get_config_general_settings( - field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict - ) - except Exception: - return [] - - pass_through_endpoint_data: Optional[List] = response.field_value - if pass_through_endpoint_data is None: - return [] - - returned_endpoints: List[PassThroughGenericEndpoint] = [] - if endpoint_id is None: - # Return all endpoints from DB, mark as not from config - for endpoint in pass_through_endpoint_data: - if isinstance(endpoint, dict): - endpoint_dict = dict(endpoint) - endpoint_dict["is_from_config"] = False - returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) - elif isinstance(endpoint, PassThroughGenericEndpoint): - endpoint_dict = endpoint.model_dump() - endpoint_dict["is_from_config"] = False - returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) - else: - # Find specific endpoint by ID - found_endpoint = _find_endpoint_by_id(pass_through_endpoint_data, endpoint_id) - if found_endpoint is not None: - endpoint_dict = ( - found_endpoint.model_dump() - if isinstance(found_endpoint, PassThroughGenericEndpoint) - else dict(found_endpoint) - ) - endpoint_dict["is_from_config"] = False - returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) - - return returned_endpoints - - -async def _filter_endpoints_by_team_allowed_routes( - team_id: str, - pass_through_endpoints: List[PassThroughGenericEndpoint], - prisma_client, -) -> List[PassThroughGenericEndpoint]: - """ - Filter pass-through endpoints based on team's allowed_passthrough_routes metadata. - - Args: - team_id: The team ID to check permissions for - pass_through_endpoints: List of endpoints to filter - prisma_client: Database client - - Returns: - Filtered list of endpoints based on team permissions - - Raises: - HTTPException: If team is not found - """ - # retrieve team from db - team = await prisma_client.db.litellm_teamtable.find_unique( - where={"team_id": team_id}, - ) - if team is None: - raise HTTPException( - status_code=404, - detail={"error": "Team not found"}, - ) - - # retrieve team metadata - team_metadata = team.metadata - if ( - team_metadata is not None - and team_metadata.get("allowed_passthrough_routes") is not None - ): - ## FILTER pass_through_endpoints by allowed_passthrough_routes - pass_through_endpoints = [ - endpoint - for endpoint in pass_through_endpoints - if endpoint.path in team_metadata.get("allowed_passthrough_routes") - ] - - return pass_through_endpoints - - -@router.get( - "/config/pass_through_endpoint", - dependencies=[Depends(user_api_key_auth)], - response_model=PassThroughEndpointResponse, -) -@router.get( - "/config/pass_through_endpoint/team/{team_id}", - dependencies=[Depends(user_api_key_auth)], - response_model=PassThroughEndpointResponse, -) -async def get_pass_through_endpoints( - endpoint_id: Optional[str] = None, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), - team_id: Optional[str] = None, -): - """ - GET configured pass through endpoint. - - If no endpoint_id given, return all configured endpoints. - """ ## Get existing pass-through endpoint field value - from litellm.proxy._types import CommonProxyErrors - from litellm.proxy.proxy_server import prisma_client - - if prisma_client is None: - raise HTTPException( - status_code=500, - detail={"error": CommonProxyErrors.db_not_connected_error.value}, - ) - - # Get endpoints from DB (editable via UI) - db_endpoints = await _get_pass_through_endpoints_from_db( - endpoint_id=endpoint_id, user_api_key_dict=user_api_key_dict - ) - - # Get endpoints from config file (read-only, not editable via UI) - config_endpoints = _get_pass_through_endpoints_from_config() - - # Merge: config endpoints not in DB + all DB endpoints (DB overrides config for same path) - db_paths = {ep.path for ep in db_endpoints} - config_only_endpoints = [ep for ep in config_endpoints if ep.path not in db_paths] - if endpoint_id is not None: - # When filtering by endpoint_id, only return if found in DB (config endpoints use generated IDs) - pass_through_endpoints = db_endpoints - else: - pass_through_endpoints = config_only_endpoints + db_endpoints - - if team_id is not None: - pass_through_endpoints = await _filter_endpoints_by_team_allowed_routes( - team_id=team_id, - pass_through_endpoints=pass_through_endpoints, - prisma_client=prisma_client, - ) - - return PassThroughEndpointResponse(endpoints=pass_through_endpoints) - - -@router.post( - "/config/pass_through_endpoint/{endpoint_id}", - dependencies=[Depends(user_api_key_auth)], -) -async def update_pass_through_endpoints( - endpoint_id: str, - data: PassThroughGenericEndpoint, - request: Request, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): - """ - Update a pass-through endpoint by ID. - """ - from litellm.proxy.proxy_server import ( - get_config_general_settings, - update_config_general_settings, - ) - - ## Get existing pass-through endpoint field value - try: - response: ConfigFieldInfo = await get_config_general_settings( - field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict - ) - except Exception: - raise HTTPException( - status_code=404, - detail={"error": "No pass-through endpoints found"}, - ) - - pass_through_endpoint_data: Optional[List] = response.field_value - if pass_through_endpoint_data is None: - raise HTTPException( - status_code=404, - detail={"error": "No pass-through endpoints found"}, - ) - - # Find the endpoint to update - found_endpoint = _find_endpoint_by_id(pass_through_endpoint_data, endpoint_id) - - if found_endpoint is None: - raise HTTPException( - status_code=404, - detail={"error": f"Endpoint with ID '{endpoint_id}' not found"}, - ) - - # Find the index for updating the list - endpoint_index = None - for idx, endpoint in enumerate(pass_through_endpoint_data): - _endpoint = ( - PassThroughGenericEndpoint(**endpoint) - if isinstance(endpoint, dict) - else endpoint - ) - if _endpoint.id == endpoint_id: - endpoint_index = idx - break - - if endpoint_index is None: - raise HTTPException( - status_code=404, - detail={ - "error": f"Could not find index for endpoint with ID '{endpoint_id}'" - }, - ) - - # Get the update data as dict, excluding None values for partial updates - # Exclude is_from_config as it's a response-only field (computed at read time) - update_data = data.model_dump(exclude_none=True, exclude={"is_from_config"}) - - # Start with existing endpoint data - endpoint_dict = found_endpoint.model_dump() - - # Update with new data (only non-None values) - endpoint_dict.update(update_data) - - # Preserve existing ID if not provided in update and endpoint has ID - if "id" not in update_data and found_endpoint.id is not None: - endpoint_dict["id"] = found_endpoint.id - - # Remove is_from_config before saving - it's a response-only field (computed at read time) - endpoint_dict.pop("is_from_config", None) - - # Create updated endpoint object - updated_endpoint = PassThroughGenericEndpoint(**endpoint_dict) - - # Update the list - pass_through_endpoint_data[endpoint_index] = endpoint_dict - - # Remove old routes from registry before they get re-registered - InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_id) - - ## Update db - updated_data = ConfigFieldUpdate( - field_name="pass_through_endpoints", - field_value=pass_through_endpoint_data, - config_type="general_settings", - ) - - await update_config_general_settings( - data=updated_data, user_api_key_dict=user_api_key_dict - ) - - # Re-register the route with updated headers - _custom_headers: Optional[dict] = updated_endpoint.headers or {} - _custom_headers = await set_env_variables_in_header(custom_headers=_custom_headers) - - if updated_endpoint.include_subpath: - InitPassThroughEndpointHelpers.add_subpath_route( - app=request.app, - path=updated_endpoint.path, - target=updated_endpoint.target, - custom_headers=_custom_headers, - forward_headers=None, # Defaults not available in model? assuming None logic handles it - merge_query_params=None, - dependencies=None, - cost_per_request=updated_endpoint.cost_per_request, - endpoint_id=updated_endpoint.id or endpoint_id or "", - guardrails=getattr(updated_endpoint, "guardrails", None), - methods=updated_endpoint.methods, - default_query_params=updated_endpoint.default_query_params, - ) - else: - InitPassThroughEndpointHelpers.add_exact_path_route( - app=request.app, - path=updated_endpoint.path, - target=updated_endpoint.target, - custom_headers=_custom_headers, - forward_headers=None, - merge_query_params=None, - dependencies=None, - cost_per_request=updated_endpoint.cost_per_request, - endpoint_id=updated_endpoint.id or endpoint_id or "", - guardrails=getattr(updated_endpoint, "guardrails", None), - methods=updated_endpoint.methods, - default_query_params=updated_endpoint.default_query_params, - ) - - return PassThroughEndpointResponse( - endpoints=[updated_endpoint] if updated_endpoint else [] - ) - - -@router.post( - "/config/pass_through_endpoint", - dependencies=[Depends(user_api_key_auth)], -) -async def create_pass_through_endpoints( - data: PassThroughGenericEndpoint, - request: Request, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): - """ - Create new pass-through endpoint - """ - from litellm._uuid import uuid - from litellm.proxy.proxy_server import ( - get_config_general_settings, - update_config_general_settings, - ) - - ## Get existing pass-through endpoint field value - - try: - response: ConfigFieldInfo = await get_config_general_settings( - field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict - ) - except Exception: - response = ConfigFieldInfo( - field_name="pass_through_endpoints", field_value=None - ) - - ## Auto-generate ID if not provided - # Exclude is_from_config as it's a response-only field (computed at read time) - data_dict = data.model_dump(exclude={"is_from_config"}) - if data_dict.get("id") is None: - data_dict["id"] = str(uuid.uuid4()) - - if response.field_value is None: - response.field_value = [data_dict] - elif isinstance(response.field_value, List): - response.field_value.append(data_dict) - - ## Update db - updated_data = ConfigFieldUpdate( - field_name="pass_through_endpoints", - field_value=response.field_value, - config_type="general_settings", - ) - await update_config_general_settings( - data=updated_data, user_api_key_dict=user_api_key_dict - ) - - # Return the created endpoint with the generated ID - created_endpoint = PassThroughGenericEndpoint(**data_dict) - - # Register the new route - _custom_headers: Optional[dict] = created_endpoint.headers or {} - _custom_headers = await set_env_variables_in_header(custom_headers=_custom_headers) - - if created_endpoint.include_subpath: - InitPassThroughEndpointHelpers.add_subpath_route( - app=request.app, - path=created_endpoint.path, - target=created_endpoint.target, - custom_headers=_custom_headers, - forward_headers=None, - merge_query_params=None, - dependencies=None, - cost_per_request=created_endpoint.cost_per_request, - endpoint_id=created_endpoint.id or "", - guardrails=getattr(created_endpoint, "guardrails", None), - methods=created_endpoint.methods, - default_query_params=created_endpoint.default_query_params, - ) - else: - InitPassThroughEndpointHelpers.add_exact_path_route( - app=request.app, - path=created_endpoint.path, - target=created_endpoint.target, - custom_headers=_custom_headers, - forward_headers=None, - merge_query_params=None, - dependencies=None, - cost_per_request=created_endpoint.cost_per_request, - endpoint_id=created_endpoint.id or "", - guardrails=getattr(created_endpoint, "guardrails", None), - methods=created_endpoint.methods, - default_query_params=created_endpoint.default_query_params, - ) - - return PassThroughEndpointResponse(endpoints=[created_endpoint]) - - -@router.delete( - "/config/pass_through_endpoint", - dependencies=[Depends(user_api_key_auth)], - response_model=PassThroughEndpointResponse, -) -async def delete_pass_through_endpoints( - endpoint_id: str, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): - """ - Delete a pass-through endpoint by ID. - - Returns - the deleted endpoint - """ - from litellm.proxy.proxy_server import ( - get_config_general_settings, - update_config_general_settings, - ) - - ## Get existing pass-through endpoint field value - - try: - response: ConfigFieldInfo = await get_config_general_settings( - field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict - ) - except Exception: - response = ConfigFieldInfo( - field_name="pass_through_endpoints", field_value=None - ) - - ## Update field by removing endpoint - pass_through_endpoint_data: Optional[List] = response.field_value - if response.field_value is None or pass_through_endpoint_data is None: - raise HTTPException( - status_code=400, - detail={"error": "There are no pass-through endpoints setup."}, - ) - - # Find the endpoint to delete - found_endpoint = _find_endpoint_by_id(pass_through_endpoint_data, endpoint_id) - - if found_endpoint is None: - raise HTTPException( - status_code=400, - detail={ - "error": "Endpoint with ID '{}' was not found in pass-through endpoint list.".format( - endpoint_id - ) - }, - ) - - # Find the index for deleting from the list - endpoint_index = None - for idx, endpoint in enumerate(pass_through_endpoint_data): - _endpoint = ( - PassThroughGenericEndpoint(**endpoint) - if isinstance(endpoint, dict) - else endpoint - ) - if _endpoint.id == endpoint_id: - endpoint_index = idx - break - - if endpoint_index is None: - raise HTTPException( - status_code=400, - detail={ - "error": f"Could not find index for endpoint with ID '{endpoint_id}'" - }, - ) - - # Remove the endpoint - pass_through_endpoint_data.pop(endpoint_index) - response_obj = found_endpoint - - # Remove routes from registry - InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_id) - - ## Update db - updated_data = ConfigFieldUpdate( - field_name="pass_through_endpoints", - field_value=pass_through_endpoint_data, - config_type="general_settings", - ) - await update_config_general_settings( - data=updated_data, user_api_key_dict=user_api_key_dict - ) - - return PassThroughEndpointResponse(endpoints=[response_obj]) - - -def _find_endpoint_by_id( - endpoints_data: List, - endpoint_id: str, -) -> Optional[PassThroughGenericEndpoint]: - """ - Find an endpoint by ID. - - Args: - endpoints_data: List of endpoint data (dicts or PassThroughGenericEndpoint objects) - endpoint_id: ID to search for - - Returns: - Found endpoint or None if not found - """ - for endpoint in endpoints_data: - _endpoint: Optional[PassThroughGenericEndpoint] = None - if isinstance(endpoint, dict): - _endpoint = PassThroughGenericEndpoint(**endpoint) - elif isinstance(endpoint, PassThroughGenericEndpoint): - _endpoint = endpoint - - # Only compare IDs to IDs - if _endpoint is not None and _endpoint.id == endpoint_id: - return _endpoint - - return None - - -async def initialize_pass_through_endpoints_in_db(): - """ - Gets all pass-through endpoints from db and initializes them in the proxy server. - """ - pass_through_endpoints = await _get_pass_through_endpoints_from_db() - await initialize_pass_through_endpoints( - pass_through_endpoints=pass_through_endpoints - ) +import ast +import asyncio +import copy +import json +import posixpath +import traceback +from base64 import b64encode +from datetime import datetime +from typing import Any, Dict, List, Mapping, Optional, Tuple, Union, cast +from urllib.parse import urlencode, urlparse + +import httpx +from fastapi import ( + APIRouter, + Depends, + FastAPI, + HTTPException, + Request, + Response, + UploadFile, + WebSocket, + status, +) +from fastapi.responses import StreamingResponse +from starlette.datastructures import UploadFile as StarletteUploadFile +from starlette.websockets import WebSocketState +from websockets.asyncio.client import connect +from websockets.exceptions import ( + ConnectionClosedError, + ConnectionClosedOK, + InvalidStatus, +) + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm._uuid import uuid +from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.llms.base_llm.managed_resources.utils import ( + resolve_passthrough_managed_id_provider, +) +from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.passthrough import BasePassthroughUtils +from litellm.proxy._types import ( + ConfigFieldInfo, + ConfigFieldUpdate, + LiteLLMRoutes, + PassThroughEndpointResponse, + PassThroughGenericEndpoint, + ProxyException, + UserAPIKeyAuth, +) +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.common_utils.http_parsing_utils import ( + _read_request_body, + _safe_get_request_headers, +) +from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.proxy.utils import get_server_root_path, normalize_route_for_root_path +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.custom_http import httpxSpecialProvider +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + EndpointType, + LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, + LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, + PassthroughStandardLoggingPayload, +) + +from .streaming_handler import PassThroughStreamingHandler +from .success_handler import PassThroughEndpointLogging + +router = APIRouter() + +pass_through_endpoint_logging = PassThroughEndpointLogging() + +# Global registry to track registered pass-through routes and prevent memory leaks +_registered_pass_through_routes: Dict[ + str, Dict[str, Union[str, List[str], Dict[str, Any]]] +] = {} + + +def get_response_body(response: httpx.Response) -> Optional[dict]: + try: + return response.json() + except Exception: + return None + + +async def set_env_variables_in_header(custom_headers: Optional[dict]) -> Optional[dict]: + """ + checks if any headers on config.yaml are defined as os.environ/COHERE_API_KEY etc + + only runs for headers defined on config.yaml + + example header can be + + {"Authorization": "Bearer os.environ/COHERE_API_KEY"} + """ + if custom_headers is None: + return None + headers = {} + for key, value in custom_headers.items(): + # langfuse Api requires base64 encoded headers - it's simpleer to just ask litellm users to set their langfuse public and secret keys + # we can then get the b64 encoded keys here + if key == "LANGFUSE_PUBLIC_KEY" or key == "LANGFUSE_SECRET_KEY": + # langfuse requires b64 encoded headers - we construct that here + _langfuse_public_key = custom_headers["LANGFUSE_PUBLIC_KEY"] + _langfuse_secret_key = custom_headers["LANGFUSE_SECRET_KEY"] + if isinstance( + _langfuse_public_key, str + ) and _langfuse_public_key.startswith("os.environ/"): + _langfuse_public_key = get_secret_str(_langfuse_public_key) + if isinstance( + _langfuse_secret_key, str + ) and _langfuse_secret_key.startswith("os.environ/"): + _langfuse_secret_key = get_secret_str(_langfuse_secret_key) + headers["Authorization"] = "Basic " + b64encode( + f"{_langfuse_public_key}:{_langfuse_secret_key}".encode("utf-8") + ).decode("ascii") + else: + # for all other headers + headers[key] = value + if isinstance(value, str) and "os.environ/" in value: + verbose_proxy_logger.debug( + "pass through endpoint - looking up 'os.environ/' variable" + ) + # get string section that is os.environ/ + start_index = value.find("os.environ/") + _variable_name = value[start_index:] + + verbose_proxy_logger.debug( + "pass through endpoint - getting secret for variable name: %s", + _variable_name, + ) + _secret_value = get_secret_str(_variable_name) + if _secret_value is not None: + new_value = value.replace(_variable_name, _secret_value) + headers[key] = new_value + return headers + + +async def chat_completion_pass_through_endpoint( # noqa: PLR0915 + fastapi_response: Response, + request: Request, + adapter_id: str, + user_api_key_dict: UserAPIKeyAuth, +): + from litellm.proxy.proxy_server import ( + add_litellm_data_to_request, + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + data = {} + try: + body = await request.body() + body_str = body.decode() + try: + data = ast.literal_eval(body_str) + except Exception: + data = json.loads(body_str) + + data["adapter_id"] = adapter_id + + verbose_proxy_logger.debug( + "Request received by LiteLLM:\n{}".format(json.dumps(data, indent=4)), + ) + data["model"] = ( + general_settings.get("completion_model", None) # server default + or user_model # model name passed via cli args + or data.get("model", None) # default passed in http request + ) + if user_model: + data["model"] = user_model + + data = await add_litellm_data_to_request( + data=data, # type: ignore + request=request, + general_settings=general_settings, + user_api_key_dict=user_api_key_dict, + version=version, + proxy_config=proxy_config, + ) + + # override with user settings, these are params passed via cli + if user_temperature: + data["temperature"] = user_temperature + if user_request_timeout: + data["request_timeout"] = user_request_timeout + if user_max_tokens: + data["max_tokens"] = user_max_tokens + if user_api_base: + data["api_base"] = user_api_base + + ### MODEL ALIAS MAPPING ### + # check if model name in model alias map + # get the actual model name + if data["model"] in litellm.model_alias_map: + data["model"] = litellm.model_alias_map[data["model"]] + + # Check key-specific aliases + if ( + isinstance(data["model"], str) + and user_api_key_dict.aliases + and isinstance(user_api_key_dict.aliases, dict) + and data["model"] in user_api_key_dict.aliases + ): + data["model"] = user_api_key_dict.aliases[data["model"]] + + ### CALL HOOKS ### - modify incoming data before calling the model + data = await proxy_logging_obj.pre_call_hook( # type: ignore + user_api_key_dict=user_api_key_dict, data=data, call_type="text_completion" + ) + + ### ROUTE THE REQUESTs ### + router_model_names = llm_router.model_names if llm_router is not None else [] + # skip router if user passed their key + if "api_key" in data: + llm_response = asyncio.create_task(litellm.aadapter_completion(**data)) + elif ( + llm_router is not None and data["model"] in router_model_names + ): # model in router model list + llm_response = asyncio.create_task(llm_router.aadapter_completion(**data)) + elif ( + llm_router is not None + and llm_router.model_group_alias is not None + and data["model"] in llm_router.model_group_alias + ): # model set in model_group_alias + llm_response = asyncio.create_task(llm_router.aadapter_completion(**data)) + elif llm_router is not None and llm_router.has_model_id( + data["model"] + ): # model in router model list + llm_response = asyncio.create_task(llm_router.aadapter_completion(**data)) + elif ( + llm_router is not None + and data["model"] not in router_model_names + and ( + llm_router.default_deployment is not None + or len(llm_router.pattern_router.patterns) > 0 + ) + ): # check for wildcard routes or default deployment before checking deployment_names + llm_response = asyncio.create_task(llm_router.aadapter_completion(**data)) + elif ( + llm_router is not None and data["model"] in llm_router.deployment_names + ): # model in router deployments, calling a specific deployment on the router (lowest priority) + llm_response = asyncio.create_task( + llm_router.aadapter_completion(**data, specific_deployment=True) + ) + elif user_model is not None: # `litellm --model ` + llm_response = asyncio.create_task(litellm.aadapter_completion(**data)) + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "error": "completion: Invalid model name passed in model=" + + data.get("model", "") + }, + ) + + # Await the llm_response task + response = await llm_response + + hidden_params = getattr(response, "_hidden_params", {}) or {} + model_id = hidden_params.get("model_id", None) or "" + cache_key = hidden_params.get("cache_key", None) or "" + api_base = hidden_params.get("api_base", None) or "" + response_cost = hidden_params.get("response_cost", None) or "" + + ### ALERTING ### + asyncio.create_task( + proxy_logging_obj.update_request_status( + litellm_call_id=data.get("litellm_call_id", ""), status="success" + ) + ) + + verbose_proxy_logger.debug("final response: %s", response) + + fastapi_response.headers.update( + ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=user_api_key_dict, + model_id=model_id, + cache_key=cache_key, + api_base=api_base, + version=version, + response_cost=response_cost, + ) + ) + + verbose_proxy_logger.debug("\nResponse from Litellm:\n{}".format(response)) + return response + except Exception as e: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data + ) + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.completion(): Exception occured - {}".format( + str(e) + ) + ) + error_msg = f"{str(e)}" + raise ProxyException( + message=getattr(e, "message", error_msg), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", 500), + ) + + +class HttpPassThroughEndpointHelpers(BasePassthroughUtils): + @staticmethod + def get_response_headers( + headers: httpx.Headers, + litellm_call_id: Optional[str] = None, + custom_headers: Optional[dict] = None, + ) -> dict: + # Exclude headers that uvicorn writes itself (server, date) and + # encoding/length headers that don't survive re-serialization. + # If we forward the upstream's Server header, uvicorn adds its + # own and strict HTTP parsers (e.g. aiohttp) reject the + # response with "Duplicate 'Server' header found". + excluded_headers = { + "transfer-encoding", + "content-encoding", + "content-length", + "server", + "date", + "connection", + "keep-alive", + } + + return_headers = { + key: value + for key, value in headers.items() + if key.lower() not in excluded_headers + } + if litellm_call_id: + return_headers["x-litellm-call-id"] = litellm_call_id + if custom_headers: + # Ensure custom headers don't override actual upstream response headers or let framework defaults (like content-length: 0) interfere. + sanitized_custom_headers = { + key: value + for key, value in custom_headers.items() + if key.lower() not in excluded_headers + } + return_headers.update(sanitized_custom_headers) + + return return_headers + + @staticmethod + def get_endpoint_type(url: str) -> EndpointType: + parsed_url = urlparse(url) + if ( + ("generateContent") in url + or ("streamGenerateContent") in url + or ("rawPredict") in url + or ("streamRawPredict") in url + ): + return EndpointType.VERTEX_AI + elif parsed_url.hostname == "api.anthropic.com": + return EndpointType.ANTHROPIC + elif ( + parsed_url.hostname == "api.openai.com" + or parsed_url.hostname == "openai.azure.com" + or (parsed_url.hostname and "openai.com" in parsed_url.hostname) + ): + return EndpointType.OPENAI + return EndpointType.GENERIC + + @staticmethod + async def _make_non_streaming_http_request( + request: Request, + async_client: httpx.AsyncClient, + url: str, + headers: dict, + requested_query_params: Optional[dict] = None, + custom_body: Optional[dict] = None, + ) -> httpx.Response: + """ + Make a non-streaming HTTP request + + If request is GET, don't include a JSON body + """ + if request.method == "GET": + response = await async_client.request( + method=request.method, + url=url, + headers=headers, + params=requested_query_params, + ) + else: + response = await async_client.request( + method=request.method, + url=url, + headers=headers, + params=requested_query_params, + json=custom_body, + ) + return response + + @staticmethod + async def non_streaming_http_request_handler( + request: Request, + async_client: httpx.AsyncClient, + url: httpx.URL, + headers: dict, + requested_query_params: Optional[dict] = None, + _parsed_body: Optional[dict] = None, + forward_multipart: bool = False, + ) -> httpx.Response: + """ + Handle non-streaming HTTP requests + + Handles special cases when GET requests, multipart/form-data requests, and generic httpx requests + """ + if request.method == "GET": + response = await async_client.request( + method=request.method, + url=url, + headers=headers, + params=requested_query_params, + ) + elif ( + HttpPassThroughEndpointHelpers.is_multipart(request) is True + and forward_multipart + ): + # Forward multipart via make_multipart_http_request even when _parsed_body is + # non-empty (pass_through_request always injects litellm_logging_obj, etc.). + # forward_multipart is False when custom_body was supplied (JSON body despite + # multipart content-type) — those requests use the generic json= path. + return await HttpPassThroughEndpointHelpers.make_multipart_http_request( + request=request, + async_client=async_client, + url=url, + headers=headers, + requested_query_params=requested_query_params, + ) + else: + # Generic httpx method + response = await async_client.request( + method=request.method, + url=url, + headers=headers, + params=requested_query_params, + json=_parsed_body, + ) + return response + + @staticmethod + def is_multipart(request: Request) -> bool: + """Check if the request is a multipart/form-data request""" + return "multipart/form-data" in request.headers.get("content-type", "") + + @staticmethod + async def _build_request_files_from_upload_file( + upload_file: Union[UploadFile, StarletteUploadFile], + ) -> Tuple[Optional[str], bytes, Optional[str]]: + """Build a request files dict from an UploadFile object""" + file_content = await upload_file.read() + return (upload_file.filename, file_content, upload_file.content_type) + + @staticmethod + async def make_multipart_http_request( + request: Request, + async_client: httpx.AsyncClient, + url: httpx.URL, + headers: dict, + requested_query_params: Optional[dict] = None, + stream: bool = False, + ) -> httpx.Response: + """Process multipart/form-data requests, handling both files and form fields""" + form_data = await request.form() + files = {} + form_data_dict = {} + + for field_name, field_value in form_data.items(): + if isinstance(field_value, (StarletteUploadFile, UploadFile)): + files[field_name] = ( + await HttpPassThroughEndpointHelpers._build_request_files_from_upload_file( + upload_file=field_value + ) + ) + else: + form_data_dict[field_name] = field_value + + # Remove content-type header - httpx will set it correctly with the new boundary + # when it creates the multipart body from files/data parameters + headers_copy = headers.copy() + headers_copy.pop("content-type", None) + + # httpx.AsyncClient.request() does not accept stream=; use send() for streaming. + if stream: + req = async_client.build_request( + request.method, + url, + headers=headers_copy, + params=requested_query_params, + files=files, + data=form_data_dict, + ) + return await async_client.send(req, stream=True) + + return await async_client.request( + method=request.method, + url=url, + headers=headers_copy, + params=requested_query_params, + files=files, + data=form_data_dict, + ) + + @staticmethod + def _init_kwargs_for_pass_through_endpoint( + request: Request, + user_api_key_dict: UserAPIKeyAuth, + passthrough_logging_payload: PassthroughStandardLoggingPayload, + logging_obj: LiteLLMLoggingObj, + _parsed_body: Optional[dict] = None, + litellm_call_id: Optional[str] = None, + ) -> dict: + """ + Filter out litellm params from the request body + """ + from litellm.types.utils import all_litellm_params + + _parsed_body = _parsed_body or {} + + litellm_params_in_body = {} + for k in all_litellm_params: + if k in _parsed_body: + litellm_params_in_body[k] = _parsed_body.pop(k, None) + + _metadata = dict( + LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( + user_api_key_dict=user_api_key_dict + ) + ) + + litellm_metadata = litellm_params_in_body.pop("litellm_metadata", None) + metadata = litellm_params_in_body.pop("metadata", None) + if litellm_metadata: + _metadata.update(litellm_metadata) + if metadata: + _metadata.update(metadata) + + _metadata = _update_metadata_with_tags_in_header( + request=request, + metadata=_metadata, + ) + + # Set internal keys after merging client-supplied metadata so a request + # body that mirrors them cannot clobber the authenticated key or the + # real parent span. + _metadata["user_api_key"] = user_api_key_dict.api_key + _metadata["litellm_parent_otel_span"] = user_api_key_dict.parent_otel_span + + kwargs = { + "litellm_params": { + **litellm_params_in_body, # type: ignore + "metadata": _metadata, + "proxy_server_request": { + "url": str(request.url), + "method": request.method, + "body": copy.copy(_parsed_body), # use copy instead of deepcopy + "headers": request.headers, + }, + }, + "call_type": "pass_through_endpoint", + "litellm_call_id": litellm_call_id, + "passthrough_logging_payload": passthrough_logging_payload, + } + + logging_obj.model_call_details["passthrough_logging_payload"] = ( + passthrough_logging_payload + ) + + return kwargs + + @staticmethod + def construct_target_url_with_subpath( + base_target: str, subpath: str, include_subpath: Optional[bool] + ) -> str: + """ + Helper function to construct the full target URL with subpath handling. + + Args: + base_target: The base target URL + subpath: The captured subpath from the request + include_subpath: Whether to include the subpath in the target URL + + Returns: + The constructed full target URL + """ + if not include_subpath: + return base_target + + if not subpath: + return base_target + + # Ensure base_target ends with / and subpath doesn't start with / + if not base_target.endswith("/"): + base_target = base_target + "/" + if subpath.startswith("/"): + subpath = subpath[1:] + + # Resolve any '..' segments in the subpath so it cannot climb above + # the base_target prefix that the operator configured. Preserve a + # trailing slash on the original subpath since some upstreams treat + # `/foo` and `/foo/` as different resources. + trailing_slash = subpath.endswith("/") + safe_subpath = posixpath.normpath("/" + subpath).lstrip("/") + if safe_subpath == ".": + safe_subpath = "" + if trailing_slash and safe_subpath and not safe_subpath.endswith("/"): + safe_subpath += "/" + + return base_target + safe_subpath + + @staticmethod + def join_base_and_endpoint_path(base_url: httpx.URL, endpoint_path: str) -> str: + """ + Combine the path component of ``base_url`` with ``endpoint_path``. + + Preserves any path prefix configured on the base URL and resolves + ``..`` segments in the endpoint so the result stays within the base + path. A trailing slash on ``endpoint_path`` is preserved. + """ + trailing_slash = endpoint_path.endswith("/") + base_path = base_url.path or "" + if not base_path or base_path == "/": + normalized_endpoint = posixpath.normpath("/" + endpoint_path.lstrip("/")) + if trailing_slash and normalized_endpoint != "/": + normalized_endpoint += "/" + return normalized_endpoint + + base_path = base_path.rstrip("/") + clean_endpoint = endpoint_path.lstrip("/") + combined = posixpath.normpath(base_path + "/" + clean_endpoint) + # If normalization climbs out of the base path, fall back to base. + if combined != base_path and not combined.startswith(base_path + "/"): + return base_path + "/" + if trailing_slash and not combined.endswith("/"): + combined += "/" + return combined + + @staticmethod + def _update_stream_param_based_on_request_body( + parsed_body: dict, + stream: Optional[bool] = None, + ) -> Optional[bool]: + """ + If stream is provided in the request body, use it. + Otherwise, use the stream parameter passed to the `pass_through_request` function + """ + if "stream" in parsed_body: + return parsed_body.get("stream", stream) + return stream + + +async def pass_through_request( # noqa: PLR0915 + request: Request, + target: str, + custom_headers: dict, + user_api_key_dict: UserAPIKeyAuth, + custom_body: Optional[dict] = None, + forward_headers: Optional[bool] = False, + merge_query_params: Optional[bool] = False, + query_params: Optional[dict] = None, + default_query_params: Optional[dict] = None, + stream: Optional[bool] = None, + cost_per_request: Optional[float] = None, + custom_llm_provider: Optional[str] = None, + guardrails_config: Optional[dict] = None, +): + """ + Pass through endpoint handler, makes the httpx request for pass-through endpoints and ensures logging hooks are called + + Args: + request: The incoming request + target: The target URL + custom_headers: The custom headers + user_api_key_dict: The user API key dictionary + custom_body: The custom body + forward_headers: Whether to forward headers + merge_query_params: Whether to merge query params + query_params: The query params + default_query_params: The default query params to be applied if not overridden by client + stream: Whether to stream the response + cost_per_request: Optional field - cost per request to the target endpoint + custom_llm_provider: Optional field - custom LLM provider for the endpoint + guardrails_config: Optional field - guardrails configuration for passthrough endpoint + """ + from litellm.exceptions import ModifyResponseException + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.pass_through_endpoints.passthrough_guardrails import ( + PassthroughGuardrailHandler, + ) + from litellm.proxy.proxy_server import proxy_logging_obj + + ######################################################### + # Initialize variables + ######################################################### + litellm_call_id = str(uuid.uuid4()) + url: Optional[httpx.URL] = None + + # parsed request body + _parsed_body: Optional[dict] = None + # kwargs for pass through endpoint, contains metadata, litellm_params, call_type, litellm_call_id, passthrough_logging_payload + kwargs: Optional[dict] = None + logging_obj: Optional[Logging] = None + + ######################################################### + try: + url = httpx.URL(target) + headers = custom_headers + headers = HttpPassThroughEndpointHelpers.forward_headers_from_request( + request_headers=_safe_get_request_headers(request).copy(), + headers=headers, + forward_headers=forward_headers, + ) + + # Apply default query parameters if provided, regardless of merge_query_params setting + if default_query_params or merge_query_params: + # Determine what to merge based on settings + request_params = dict(request.query_params) if merge_query_params else {} + + # Create a new URL with the merged query params + url = url.copy_with( + query=urlencode( + HttpPassThroughEndpointHelpers.get_merged_query_parameters( + existing_url=url, + request_query_params=request_params, + default_query_params=default_query_params, + ) + ).encode("ascii") + ) + + endpoint_type: EndpointType = HttpPassThroughEndpointHelpers.get_endpoint_type( + str(url) + ) + + # SigV4-signed callers (e.g. Bedrock) attach the exact bytes that were + # signed via request.state; we must send those instead of re-encoding the + # parsed dict (hooks mutate it, breaking the signature / Content-Length). + # Tolerate request objects without `state` (test fixtures) and only honor + # values httpx accepts for `content=`. + _request_state = getattr(request, "state", None) + state_raw_body: Optional[Union[str, bytes]] = ( + getattr(_request_state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, None) + if _request_state is not None + else None + ) + if state_raw_body is not None and not isinstance( + state_raw_body, (str, bytes, bytearray) + ): + state_raw_body = None + + # Skip body parsing for multipart requests - make_multipart_http_request will handle it + # But if custom_body is provided (e.g., JSON parsed despite multipart content-type), use it + is_multipart = ( + HttpPassThroughEndpointHelpers.is_multipart(request) and not custom_body + ) + + if custom_body: + _parsed_body = custom_body + elif is_multipart: + # Don't parse multipart body here - it will be handled by make_multipart_http_request + _parsed_body = {} + else: + _parsed_body = await _read_request_body(request) + verbose_proxy_logger.debug( + "Pass through endpoint sending request to \nURL {}\nheaders: {}\nbody: {}\n".format( + url, headers, _parsed_body + ) + ) + + ### COLLECT GUARDRAILS FOR PASSTHROUGH ENDPOINT ### + # Passthrough endpoints are opt-in only for guardrails + # When enabled, collect guardrails from org/team/key levels + passthrough-specific + guardrails_to_run = PassthroughGuardrailHandler.collect_guardrails( + user_api_key_dict=user_api_key_dict, + passthrough_guardrails_config=guardrails_config, + ) + + # Add guardrails to metadata if any should run + if guardrails_to_run and len(guardrails_to_run) > 0: + if _parsed_body is None: + _parsed_body = {} + if "metadata" not in _parsed_body: + _parsed_body["metadata"] = {} + _parsed_body["metadata"]["guardrails"] = guardrails_to_run + verbose_proxy_logger.debug( + f"Added guardrails to passthrough request metadata: {guardrails_to_run}" + ) + + ## LOGGING OBJECT ## - initialize before pre_call_hook so guardrails can access it + # Surface the requested model (when the body carries one) so logging/spans + # read e.g. ``chat gpt-4o`` instead of ``chat unknown``. + passthrough_model = ( + _parsed_body.get("model") if isinstance(_parsed_body, dict) else None + ) or "unknown" + start_time = datetime.now() + logging_obj = Logging( + model=passthrough_model, + messages=[{"role": "user", "content": safe_dumps(_parsed_body)}], + stream=False, + call_type="pass_through_endpoint", + start_time=start_time, + litellm_call_id=litellm_call_id, + function_id="1245", + ) + + # Store passthrough guardrails config on logging_obj for field targeting + logging_obj.passthrough_guardrails_config = guardrails_config + + # Store logging_obj in data so guardrails can access it + if _parsed_body is None: + _parsed_body = {} + _parsed_body["litellm_logging_obj"] = logging_obj + + ### CALL HOOKS ### - modify incoming data / reject request before calling the model + _parsed_body = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=user_api_key_dict, + data=_parsed_body, + call_type="pass_through_endpoint", + ) + async_client_obj = get_async_httpx_client( + llm_provider=httpxSpecialProvider.PassThroughEndpoint, + params={"timeout": 600}, + ) + async_client = async_client_obj.client + passthrough_logging_payload = PassthroughStandardLoggingPayload( + url=str(url), + request_body=_parsed_body, + request_method=getattr(request, "method", None), + cost_per_request=cost_per_request, + ) + kwargs = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( + user_api_key_dict=user_api_key_dict, + _parsed_body=_parsed_body, + passthrough_logging_payload=passthrough_logging_payload, + litellm_call_id=litellm_call_id, + request=request, + logging_obj=logging_obj, + ) + + # Store custom_llm_provider in kwargs and logging object if provided + if custom_llm_provider: + logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider + logging_obj.model_call_details["litellm_params"] = kwargs.get( + "litellm_params", {} + ) + + # done for supporting 'parallel_request_limiter.py' with pass-through endpoints + logging_obj.update_environment_variables( + model=passthrough_model, + user="unknown", + optional_params={}, + litellm_params=kwargs["litellm_params"], + call_type="pass_through_endpoint", + ) + logging_obj.model_call_details["litellm_call_id"] = litellm_call_id + + # combine url with query params for logging + requested_query_params: Optional[dict] = query_params or dict( + request.query_params + ) + + ## PASSTHROUGH MANAGED ID RESOLUTION (INPUT) ## + # Resolve managed IDs in path, query params, and body back to raw + # provider IDs before forwarding upstream. Gated by feature flag and + # enterprise managed-files hook. Runs after pre_call_hook so + # guardrails have already seen the managed IDs. + from litellm.proxy.proxy_server import ( + general_settings as proxy_general_settings, + ) + + _managed_id_provider = resolve_passthrough_managed_id_provider( + custom_llm_provider + ) + + if ( + proxy_general_settings.get("passthrough_managed_object_ids", False) + and _managed_id_provider is not None + ): + verbose_proxy_logger.debug( + "pass_through_endpoint: managed-id input rewrite enabled for route=%s method=%s", + request.url.path, + request.method, + ) + _passthrough_managed_hook = proxy_logging_obj.get_proxy_hook( + "managed_files" + ) + if _passthrough_managed_hook is not None: + from litellm.proxy.pass_through_endpoints.managed_id_rewriter import ( + rewrite_body_ids, + rewrite_path_ids, + rewrite_query_ids, + ) + from litellm.proxy.proxy_server import ( + prisma_client as _passthrough_prisma, + ) + + _original_path = url.path + _original_query_params = requested_query_params + _original_body = _parsed_body + _new_path = await rewrite_path_ids( + url.path, + _managed_id_provider, + user_api_key_dict, + _passthrough_prisma, + _passthrough_managed_hook, + ) + if _new_path != url.path: + url = url.copy_with(path=_new_path) + requested_query_params = await rewrite_query_ids( + requested_query_params, + _managed_id_provider, + user_api_key_dict, + _passthrough_prisma, + _passthrough_managed_hook, + ) + _parsed_body = await rewrite_body_ids( + _parsed_body, + _managed_id_provider, + user_api_key_dict, + _passthrough_prisma, + _passthrough_managed_hook, + ) + verbose_proxy_logger.debug( + "pass_through_endpoint: managed-id input rewrite results path_changed=%s query_changed=%s body_changed=%s route=%s method=%s", + _new_path != _original_path, + requested_query_params is not _original_query_params, + _parsed_body is not _original_body, + request.url.path, + request.method, + ) + else: + verbose_proxy_logger.debug( + "pass_through_endpoint: managed-id input rewrite skipped (managed_files hook not available) route=%s method=%s", + request.url.path, + request.method, + ) + + ## PASSTHROUGH MANAGED LIST (DB-only response) ## + # For GET /v1/files and GET /v1/batches passthrough routes, serve the + # listing entirely from our DB so each caller only sees their own IDs. + # Admins / master-key callers see all rows. Gated on the same + # conditions as INPUT/OUTPUT rewrite: feature flag, provider, AND + # the managed_files hook must be present. Without the hook no managed + # IDs are ever minted or stored, so the DB is empty and intercepting + # the list would silently hide the caller's real upstream files/batches. + if ( + proxy_general_settings.get("passthrough_managed_object_ids", False) + and _managed_id_provider is not None + and request.method == "GET" + and proxy_logging_obj.get_proxy_hook("managed_files") is not None + ): + from litellm.proxy.auth.auth_utils import get_request_route + from litellm.proxy.pass_through_endpoints.managed_id_rewriter import ( + is_passthrough_list_route, + list_passthrough_ids_from_db, + ) + from litellm.proxy.proxy_server import ( + prisma_client as _list_prisma, + ) + + if ( + is_passthrough_list_route( + _managed_id_provider, request.method, get_request_route(request) + ) + and _list_prisma is not None + ): + _list_result = await list_passthrough_ids_from_db( + provider=_managed_id_provider, + route=get_request_route(request), + user_api_key_dict=user_api_key_dict, + prisma_client=_list_prisma, + query_params=dict(request.query_params), + ) + if _list_result is not None: + verbose_proxy_logger.debug( + "pass_through_endpoint: list served from DB route=%s count=%d", + request.url.path, + len(_list_result.get("data", [])), + ) + return Response( + content=json.dumps(_list_result), + status_code=200, + media_type="application/json", + ) + + requested_query_params_str = None + if requested_query_params: + requested_query_params_str = "&".join( + f"{k}={v}" for k, v in requested_query_params.items() + ) + + logging_url = str(url) + if requested_query_params_str: + if "?" in str(url): + logging_url = str(url) + "&" + requested_query_params_str + else: + logging_url = str(url) + "?" + requested_query_params_str + + logging_obj.pre_call( + input=[{"role": "user", "content": safe_dumps(_parsed_body)}], + api_key="", + additional_args={ + "complete_input_dict": _parsed_body, + "api_base": str(logging_url), + "headers": headers, + }, + ) + stream = ( + HttpPassThroughEndpointHelpers._update_stream_param_based_on_request_body( + parsed_body=_parsed_body or {}, + stream=stream, + ) + ) + + if stream: + if is_multipart: + response = ( + await HttpPassThroughEndpointHelpers.make_multipart_http_request( + request=request, + async_client=async_client, + url=url, + headers=headers, + requested_query_params=requested_query_params, + stream=True, + ) + ) + else: + # SigV4-signed callers (Bedrock) supply the exact pre-signed bytes; + # otherwise httpx encodes the parsed JSON dict as before. + body_kwargs: Dict[str, Any] = ( + {"content": state_raw_body} + if state_raw_body is not None + else {"json": _parsed_body} + ) + req = async_client.build_request( + request.method, + url, + params=requested_query_params, + headers=headers, + **body_kwargs, + ) + + response = await async_client.send(req, stream=stream) + + try: + response.raise_for_status() + except httpx.HTTPStatusError as e: + raise HTTPException( + status_code=e.response.status_code, detail=await e.response.aread() + ) + + return StreamingResponse( + PassThroughStreamingHandler.chunk_processor( + response=response, + request_body=_parsed_body, + litellm_logging_obj=logging_obj, + endpoint_type=endpoint_type, + start_time=start_time, + passthrough_success_handler_obj=pass_through_endpoint_logging, + url_route=str(url), + ), + headers=HttpPassThroughEndpointHelpers.get_response_headers( + headers=response.headers, + litellm_call_id=litellm_call_id, + ), + status_code=response.status_code, + ) + + if state_raw_body is not None: + # SigV4-signed callers (Bedrock) require the exact pre-signed bytes + # to be forwarded so the signature/Content-Length stay valid. + response = await async_client.request( + method=request.method, + url=url, + headers=headers, + params=requested_query_params, + content=state_raw_body, + ) + else: + response = ( + await HttpPassThroughEndpointHelpers.non_streaming_http_request_handler( + request=request, + async_client=async_client, + url=url, + headers=headers, + requested_query_params=requested_query_params, + _parsed_body=_parsed_body, + forward_multipart=is_multipart, + ) + ) + verbose_proxy_logger.debug("response.headers= %s", response.headers) + + if _is_streaming_response(response) is True: + try: + response.raise_for_status() + except httpx.HTTPStatusError as e: + raise HTTPException( + status_code=e.response.status_code, detail=await e.response.aread() + ) + + return StreamingResponse( + PassThroughStreamingHandler.chunk_processor( + response=response, + request_body=_parsed_body, + litellm_logging_obj=logging_obj, + endpoint_type=endpoint_type, + start_time=start_time, + passthrough_success_handler_obj=pass_through_endpoint_logging, + url_route=str(url), + ), + headers=HttpPassThroughEndpointHelpers.get_response_headers( + headers=response.headers, + litellm_call_id=litellm_call_id, + ), + status_code=response.status_code, + ) + + try: + response.raise_for_status() + except httpx.HTTPStatusError as e: + raise HTTPException( + status_code=e.response.status_code, detail=e.response.text + ) + + if response.status_code >= 300: + raise HTTPException(status_code=response.status_code, detail=response.text) + + content = await response.aread() + + ## POST-CALL GUARDRAILS ## + _content_modified = False + response_body: Optional[dict] = get_response_body(response) + if response_body is not None and guardrails_to_run: + # Build an enriched data dict: _parsed_body has been stripped of + # `metadata` by both pre_call_hook and _init_kwargs_for_pass_through_endpoint, + # so we re-attach the configured guardrails here so should_run_guardrail + # sees them. + hook_data = dict(_parsed_body or {}) + existing_metadata = hook_data.get("metadata") + if not isinstance(existing_metadata, dict): + existing_metadata = {} + hook_data["metadata"] = { + **existing_metadata, + "guardrails": guardrails_to_run, + } + response_body = await proxy_logging_obj.post_call_success_hook( + data=hook_data, + user_api_key_dict=user_api_key_dict, + response=response_body, # type: ignore[arg-type] + ) + if isinstance(response_body, dict): + content = json.dumps(response_body).encode("utf-8") + _content_modified = True + else: + verbose_proxy_logger.debug( + "pass_through_endpoint: post_call_success_hook returned %s, expected dict — using original response", + type(response_body).__name__, + ) + elif response_body is None: + verbose_proxy_logger.debug( + "pass_through_endpoint: response body not JSON-parseable, skipping post-call guardrails" + ) + + ## PASSTHROUGH MANAGED ID MINTING (OUTPUT) ## + # Mint managed IDs for raw provider IDs in the response body and swap + # them before the response reaches the client. Runs after guardrails + # so guardrails see the raw IDs (cleaner) and the client receives the + # managed IDs. Gated by feature flag and enterprise managed-files hook. + if ( + proxy_general_settings.get("passthrough_managed_object_ids", False) + and _managed_id_provider is not None + and isinstance(response_body, dict) + and response.status_code < 300 + ): + verbose_proxy_logger.debug( + "pass_through_endpoint: managed-id output rewrite enabled for route=%s method=%s status=%s", + request.url.path, + request.method, + response.status_code, + ) + _passthrough_managed_hook = proxy_logging_obj.get_proxy_hook( + "managed_files" + ) + if _passthrough_managed_hook is not None: + from litellm.proxy.auth.auth_utils import get_request_route + from litellm.proxy.pass_through_endpoints.managed_id_rewriter import ( + rewrite_response_ids, + ) + from litellm.proxy.proxy_server import ( + prisma_client as _passthrough_prisma, + ) + + _new_body = await rewrite_response_ids( + provider=_managed_id_provider, + method=request.method, + route=get_request_route(request), + body=response_body, + user_api_key_dict=user_api_key_dict, + prisma_client=_passthrough_prisma, + managed_files_hook=_passthrough_managed_hook, + ) + if _new_body is not response_body: + response_body = _new_body + content = json.dumps(response_body).encode("utf-8") + _content_modified = True + verbose_proxy_logger.debug( + "pass_through_endpoint: managed-id output rewrite applied route=%s method=%s", + request.url.path, + request.method, + ) + else: + verbose_proxy_logger.debug( + "pass_through_endpoint: managed-id output rewrite no-op route=%s method=%s", + request.url.path, + request.method, + ) + else: + verbose_proxy_logger.debug( + "pass_through_endpoint: managed-id output rewrite skipped (managed_files hook not available) route=%s method=%s", + request.url.path, + request.method, + ) + + ## LOG SUCCESS + passthrough_logging_payload["response_body"] = response_body + end_time = datetime.now() + asyncio.create_task( + pass_through_endpoint_logging.pass_through_async_success_handler( + httpx_response=response, + response_body=response_body, + url_route=str(url), + result="", + start_time=start_time, + end_time=end_time, + logging_obj=logging_obj, + cache_hit=False, + request_body=_parsed_body or {}, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) + ) + + ## CUSTOM HEADERS - `x-litellm-*` + custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=user_api_key_dict, + call_id=litellm_call_id, + model_id=None, + cache_key=None, + api_base=str(url._uri_reference), + ) + + response_headers = HttpPassThroughEndpointHelpers.get_response_headers( + headers=response.headers, + custom_headers=custom_headers, + ) + if _content_modified: + response_headers.pop("content-length", None) + + return Response( + content=content, + status_code=response.status_code, + headers=response_headers, + ) + except ModifyResponseException as e: + verbose_proxy_logger.info( + "pass_through_endpoint: Guardrail %s modified response: %s", + e.guardrail_name, + str(e.message or "")[:200], + ) + try: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=e, + request_data=e.request_data, + ) + except Exception: + verbose_proxy_logger.warning( + "pass_through_endpoint: post_call_failure_hook raised during guardrail block", + exc_info=True, + ) + error_body = { + "error": { + "message": e.message or "Response blocked by guardrail", + "type": "content_filter", + "guardrail_name": e.guardrail_name, + "model": e.model, + } + } + return Response( + content=json.dumps(error_body), + status_code=200, + media_type="application/json", + ) + except Exception as e: + custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=user_api_key_dict, + call_id=litellm_call_id, + model_id=None, + cache_key=None, + api_base=str(url._uri_reference) if url else None, + ) + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.pass_through_endpoint(): Exception occured - {}".format( + str(e) + ) + ) + + ######################################################### + # Monitoring: Trigger post_call_failure_hook + # for pass through endpoint failure + ######################################################### + request_payload: dict = _parsed_body or {} + # add user_api_key_dict, litellm_call_id, passthrough_logging_payloa for logging + if kwargs: + for key, value in kwargs.items(): + request_payload[key] = value + if logging_obj is not None: + request_payload["litellm_logging_obj"] = logging_obj + + if ( + "model" not in request_payload + and _parsed_body + and isinstance(_parsed_body, dict) + ): + request_payload["model"] = _parsed_body.get("model", "") + if "custom_llm_provider" not in request_payload and custom_llm_provider: + request_payload["custom_llm_provider"] = custom_llm_provider + + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=e, + request_data=request_payload, + traceback_str=traceback.format_exc( + limit=MAXIMUM_TRACEBACK_LINES_TO_LOG, + ), + ) + + ######################################################### + + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "message", str(getattr(e, "detail", str(e)))), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), + headers=custom_headers, + ) + else: + error_msg = f"{str(e)}" + raise ProxyException( + message=getattr(e, "message", error_msg), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", 500), + headers=custom_headers, + ) + + +def _update_metadata_with_tags_in_header(request: Request, metadata: dict) -> dict: + """ + If tags are in the request headers, add them to the metadata + + Used for google and vertex JS SDKs, and Azure passthrough + Checks both 'tags' and 'x-litellm-tags' headers + """ + tags_to_add = [] + + # Check for 'tags' header first + _tags = request.headers.get("tags") + if _tags: + tags_to_add.extend([tag.strip() for tag in _tags.split(",")]) + + _tags = request.headers.get("x-litellm-tags") + if _tags: + tags_to_add.extend([tag.strip() for tag in _tags.split(",")]) + + # Only add tags key if there are tags to add + if tags_to_add: + if "tags" not in metadata: + metadata["tags"] = [] + metadata["tags"].extend(tags_to_add) + + return metadata + + +async def _parse_request_data_by_content_type( + request: Request, +) -> Tuple[Optional[Any], Optional[Any], Optional[Any], Optional[Any]]: + """ + Parse request data based on content type. + + Handles JSON, multipart/form-data, and URL-encoded form data. + + Returns: + Tuple of (query_params_data, custom_body_data, file_data, stream) + """ + content_type = request.headers.get("content-type", "") + + query_params_data = None + custom_body_data = None + file_data = None + stream = None + + if "application/json" in content_type: + # ✅ Handle JSON + try: + body = await request.json() + query_params_data = body.get("query_params") + custom_body_data = body.get("custom_body") + stream = body.get("stream") + except json.JSONDecodeError: + # Handle requests with no body (e.g., DELETE requests) + pass + elif "multipart/form-data" in content_type: + # ✅ Try to parse as JSON first (handles misconfigured clients sending JSON with multipart content-type) + # If that fails, skip parsing - pass_through_request will handle actual multipart + try: + body = await request.json() + # Successfully parsed as JSON - treat as JSON body + query_params_data = body.get("query_params") + custom_body_data = body.get("custom_body") + stream = body.get("stream") + # If custom_body is not set, use the entire body + if custom_body_data is None and body: + custom_body_data = body + except (json.JSONDecodeError, Exception): + # Not JSON - this is actual multipart data + # Skip parsing here to avoid consuming the request body stream + # make_multipart_http_request will handle it + pass + + elif "application/x-www-form-urlencoded" in content_type: + # ✅ Handle URL-encoded form data + form = await request.form() + query_params_data = form.get("query_params") + custom_body_data = form.get("custom_body") + + else: + # ✅ Fallback: maybe no body, just query params + query_params_data = dict(request.query_params) or None + + return query_params_data, custom_body_data, file_data, stream + + +def create_pass_through_route( + endpoint, + target: str, + custom_headers: Optional[Mapping[str, Any]] = None, + _forward_headers: Optional[bool] = False, + _merge_query_params: Optional[bool] = False, + dependencies: Optional[List] = None, + include_subpath: Optional[bool] = False, + cost_per_request: Optional[float] = None, + custom_llm_provider: Optional[str] = None, + is_streaming_request: Optional[bool] = False, + query_params: Optional[dict] = None, + default_query_params: Optional[dict] = None, + guardrails: Optional[Dict[str, Any]] = None, + config_file_path: Optional[str] = None, +): + # check if target is an adapter.py or a url + from litellm._uuid import uuid + from litellm.proxy.types_utils.utils import get_instance_fn + + try: + if isinstance(target, CustomLogger): + adapter = target + else: + adapter = get_instance_fn(value=target, config_file_path=config_file_path) + adapter_id = str(uuid.uuid4()) + litellm.adapters = [{"id": adapter_id, "adapter": adapter}] + + async def endpoint_func( # type: ignore + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + subpath: str = "", # captures sub-paths when include_subpath=True + ): + return await chat_completion_pass_through_endpoint( + fastapi_response=fastapi_response, + request=request, + adapter_id=adapter_id, + user_api_key_dict=user_api_key_dict, + ) + + except Exception: + verbose_proxy_logger.debug("Defaulting to target being a url.") + + async def endpoint_func( # type: ignore + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + subpath: str = "", # captures sub-paths when include_subpath=True + ): + from litellm.proxy.auth.auth_utils import ( # noqa: PLC0415 + get_request_route, + ) + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + ) + + path = get_request_route(request) + + # Parse request data based on content type + ( + query_params_data, + custom_body_data, + file_data, + stream, + ) = await _parse_request_data_by_content_type(request) + + if not InitPassThroughEndpointHelpers.is_registered_pass_through_route( + route=path + ): + raise HTTPException( + status_code=404, + detail=f"Pass-through endpoint {endpoint} not found. This could have been deleted or not yet added to the proxy.", + ) + + passthrough_params = ( + InitPassThroughEndpointHelpers.get_registered_pass_through_route( + route=path, method=request.method + ) + ) + target_params = { + "target": target, + "custom_headers": custom_headers, + "forward_headers": _forward_headers, + "merge_query_params": _merge_query_params, + "cost_per_request": cost_per_request, + "guardrails": None, + } + + if passthrough_params is not None: + target_params.update(passthrough_params.get("passthrough_params", {})) + + # Extract and cast parameters with proper types + param_target = target_params.get("target") or target + param_custom_headers = target_params.get("custom_headers", custom_headers) + param_forward_headers = target_params.get( + "forward_headers", _forward_headers + ) + param_merge_query_params = target_params.get( + "merge_query_params", _merge_query_params + ) + param_cost_per_request = target_params.get( + "cost_per_request", cost_per_request + ) + param_guardrails = target_params.get("guardrails", None) + param_default_query_params = target_params.get("default_query_params", None) + + # Construct the full target URL with subpath if needed + full_target = ( + HttpPassThroughEndpointHelpers.construct_target_url_with_subpath( + base_target=cast(str, param_target), + subpath=subpath, + include_subpath=include_subpath, + ) + ) + + # Ensure custom_headers is a dict. Botocore returns a HeadersDict + # for SigV4-prepared requests, which is a Mapping but not a dict. + headers_dict = ( + dict(param_custom_headers) + if isinstance(param_custom_headers, Mapping) + else {} + ) + + # Ensure query_params and custom_body are dicts or None + final_query_params = ( + query_params_data if isinstance(query_params_data, dict) else {} + ) + if query_params: + final_query_params.update(query_params) + # Programmatic callers set LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY on + # request.state (see Bedrock proxy). Parsed JSON envelope otherwise. + state_custom_body: Optional[dict] = getattr( + request.state, + LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, + None, + ) + final_custom_body: Optional[dict] = None + if isinstance(state_custom_body, dict): + final_custom_body = state_custom_body + elif isinstance(custom_body_data, dict): + final_custom_body = custom_body_data + + try: + return await pass_through_request( # type: ignore + request=request, + target=full_target, + custom_headers=headers_dict, + user_api_key_dict=user_api_key_dict, + forward_headers=cast(Optional[bool], param_forward_headers), + merge_query_params=cast(Optional[bool], param_merge_query_params), + query_params=final_query_params, + default_query_params=cast( + Optional[dict], param_default_query_params + ), + stream=is_streaming_request or stream, + custom_body=final_custom_body, + cost_per_request=cast(Optional[float], param_cost_per_request), + custom_llm_provider=custom_llm_provider, + guardrails_config=cast(Optional[dict], param_guardrails), + ) + finally: + if hasattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY): + delattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY) + if hasattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY): + delattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY) + + return endpoint_func + + +def create_websocket_passthrough_route( + endpoint: str, + target: str, + custom_headers: Optional[dict] = None, + _forward_headers: Optional[bool] = False, + dependencies: Optional[List] = None, + cost_per_request: Optional[float] = None, +): + """ + Create a WebSocket passthrough route function. + + Args: + endpoint: The endpoint path (for logging purposes) + target: The target WebSocket URL (e.g., "wss://api.example.com/ws") + custom_headers: Custom headers to include in the WebSocket connection + _forward_headers: Whether to forward incoming headers + dependencies: FastAPI dependencies to inject + + Returns: + A WebSocket passthrough function that can be registered with app.websocket() + """ + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket + + async def websocket_endpoint_func( + websocket: WebSocket, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket), + **kwargs, # For additional query parameters + ): + """ + WebSocket passthrough endpoint function. + + This function handles the WebSocket connection by: + 1. Accepting the incoming WebSocket connection + 2. Establishing a connection to the target WebSocket + 3. Forwarding messages bidirectionally + 4. Handling connection cleanup + """ + return await websocket_passthrough_request( + websocket=websocket, + target=target, + custom_headers=custom_headers or {}, + user_api_key_dict=user_api_key_dict, + forward_headers=_forward_headers, + endpoint=endpoint, + cost_per_request=cost_per_request, + accept_websocket=True, # Generic usage should accept the WebSocket + ) + + return websocket_endpoint_func + + +async def websocket_passthrough_request( # noqa: PLR0915 + websocket: WebSocket, + target: str, + custom_headers: dict, + user_api_key_dict: UserAPIKeyAuth, + forward_headers: Optional[bool] = False, + endpoint: Optional[str] = None, + cost_per_request: Optional[float] = None, + accept_websocket: bool = True, +): + """ + WebSocket passthrough request handler. + + Args: + websocket: The incoming WebSocket connection + target: The target WebSocket URL + custom_headers: Custom headers to include in the connection + user_api_key_dict: The user API key dictionary + forward_headers: Whether to forward incoming headers + endpoint: The endpoint path (for logging purposes) + cost_per_request: Optional field - cost per request to the target endpoint + """ + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.proxy_server import proxy_logging_obj + from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + PassthroughStandardLoggingPayload, + ) + + # Initialize tracking variables + start_time = datetime.now() + websocket_messages: list[dict[str, Any]] = [] + litellm_call_id = str(uuid.uuid4()) + + verbose_proxy_logger.info( + f"WebSocket passthrough ({endpoint}): Starting WebSocket connection to {target}" + ) + + # Only accept the WebSocket if requested (for generic usage) + if accept_websocket: + await websocket.accept() + verbose_proxy_logger.debug( + f"WebSocket passthrough ({endpoint}): WebSocket connection accepted" + ) + + # Prepare headers for the upstream connection + upstream_headers = custom_headers.copy() + + if forward_headers: + # Forward relevant headers from the incoming request + incoming_headers = dict(websocket.headers) + for header_name, header_value in incoming_headers.items(): + # Only forward certain headers to avoid conflicts + if header_name.lower() in [ + "authorization", + "x-api-key", + "x-goog-user-project", + ]: + upstream_headers[header_name] = header_value + + # Initialize logging object similar to HTTP passthrough + logging_obj = Logging( + model="unknown", + messages=[{"role": "user", "content": "WebSocket connection"}], + stream=True, # WebSockets are inherently streaming + call_type="pass_through_endpoint", + start_time=start_time, + litellm_call_id=litellm_call_id, + function_id="websocket_passthrough", + ) + + # Create passthrough logging payload + passthrough_logging_payload = PassthroughStandardLoggingPayload( + url=target, + request_body={}, # WebSocket doesn't have a traditional request body + request_method="WEBSOCKET", + cost_per_request=cost_per_request, + ) + + # Create a dummy request object for WebSocket connections to maintain compatibility + # with the existing _init_kwargs_for_pass_through_endpoint function + class DummyRequest: + def __init__( + self, url: str, method: str = "WEBSOCKET", headers: Optional[dict] = None + ): + self.url = url + self.method = method + self.headers = headers or {} + + def __str__(self): + return f"DummyRequest(url={self.url}, method={self.method})" + + dummy_request = DummyRequest( + url=target, + method="WEBSOCKET", + headers=dict(websocket.headers) if hasattr(websocket, "headers") else {}, + ) + + # Initialize kwargs for logging using the same pattern as HTTP passthrough + kwargs = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( + user_api_key_dict=user_api_key_dict, + _parsed_body={}, # WebSocket doesn't have a traditional request body + passthrough_logging_payload=passthrough_logging_payload, + litellm_call_id=litellm_call_id, + request=dummy_request, # type: ignore + logging_obj=logging_obj, + ) + + # Update logging environment variables + logging_obj.update_environment_variables( + model="unknown", + user="unknown", + optional_params={}, + litellm_params=dict(kwargs.get("litellm_params", {})), + call_type="pass_through_endpoint", + ) + logging_obj.model_call_details["litellm_call_id"] = litellm_call_id + + # Pre-call logging + logging_obj.pre_call( + input=[{"role": "user", "content": "WebSocket connection"}], + api_key="", + additional_args={ + "complete_input_dict": {}, + "api_base": target, + "headers": upstream_headers, + }, + ) + + ### CALL HOOKS ### - modify incoming data / reject request before calling the model + websocket_data: dict[str, Any] = {} + websocket_data = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=user_api_key_dict, + data=websocket_data, + call_type="pass_through_endpoint", + ) + + try: + verbose_proxy_logger.debug( + f"WebSocket passthrough ({endpoint}): Establishing upstream connection to {target}" + ) + async with connect( + target, + additional_headers=upstream_headers, + ) as upstream_ws: + verbose_proxy_logger.info( + f"WebSocket passthrough ({endpoint}): Upstream connection established successfully" + ) + + async def forward_client_to_upstream() -> None: + """Forward messages from client to upstream WebSocket""" + try: + while True: + message = await websocket.receive() + message_type = message.get("type") + if message_type == "websocket.disconnect": + await upstream_ws.close() + break + + text_data = message.get("text") + bytes_data = message.get("bytes") + + if text_data is not None: + # Try to extract model from client setup message for Vertex AI Live + if endpoint and "/vertex_ai/live" in endpoint: + verbose_proxy_logger.debug( + f"WebSocket passthrough ({endpoint}): Processing client message for model extraction" + ) + try: + client_message = json.loads(text_data) + if ( + isinstance(client_message, dict) + and "setup" in client_message + ): + setup_data = client_message["setup"] + verbose_proxy_logger.debug( + f"WebSocket passthrough ({endpoint}): Found setup data in client message: {setup_data}" + ) + if ( + isinstance(setup_data, dict) + and "model" in setup_data + ): + extracted_model = ( + _extract_model_from_vertex_ai_setup( + setup_data + ) + ) + if extracted_model: + kwargs["model"] = extracted_model + kwargs["custom_llm_provider"] = ( + "vertex_ai-language-models" + ) + # Update logging object with correct model + logging_obj.model = extracted_model + logging_obj.model_call_details[ + "model" + ] = extracted_model + logging_obj.model_call_details[ + "custom_llm_provider" + ] = "vertex_ai" + verbose_proxy_logger.info( + f"WebSocket passthrough ({endpoint}): Successfully extracted model '{extracted_model}' and set provider to 'vertex_ai' from client setup message" + ) + else: + verbose_proxy_logger.warning( + f"WebSocket passthrough ({endpoint}): Failed to extract model from client setup data: {setup_data}" + ) + else: + verbose_proxy_logger.debug( + f"WebSocket passthrough ({endpoint}): Setup data does not contain model field: {setup_data}" + ) + else: + verbose_proxy_logger.debug( + f"WebSocket passthrough ({endpoint}): Client message does not contain setup data" + ) + except (json.JSONDecodeError, KeyError, TypeError) as e: + verbose_proxy_logger.debug( + f"WebSocket passthrough ({endpoint}): Client message is not a valid setup message: {e}" + ) + pass # Not a JSON message or doesn't contain setup data + + await upstream_ws.send(text_data) + elif bytes_data is not None: + await upstream_ws.send(bytes_data) + except asyncio.CancelledError: + raise + except Exception: + verbose_proxy_logger.exception( + f"WebSocket passthrough ({endpoint}): error forwarding client message" + ) + await upstream_ws.close() + + async def forward_upstream_to_client() -> None: + """Forward messages from upstream to client WebSocket""" + try: + # Wait for the first response from upstream + raw_response = await upstream_ws.recv(decode=False) + # Ensure raw_response is bytes before decoding + if isinstance(raw_response, str): + raw_response = raw_response.encode("ascii") + setup_response = json.loads(raw_response.decode("ascii")) + verbose_proxy_logger.debug(f"Setup response: {setup_response}") + + # Extract model and provider from setup response for Vertex AI Live + if endpoint and "/vertex_ai/live" in endpoint: + verbose_proxy_logger.debug( + f"WebSocket passthrough ({endpoint}): Processing server setup response for model extraction" + ) + extracted_model = _extract_model_from_vertex_ai_setup( + setup_response + ) + if extracted_model: + kwargs["model"] = extracted_model + kwargs["custom_llm_provider"] = "vertex_ai_language_models" + # Update logging object with correct model + logging_obj.model = extracted_model + logging_obj.model_call_details["model"] = extracted_model + logging_obj.model_call_details["custom_llm_provider"] = ( + "vertex_ai_language_models" + ) + verbose_proxy_logger.debug( + f"WebSocket passthrough ({endpoint}): Successfully extracted model '{extracted_model}' and set provider to 'vertex_ai' from server setup response" + ) + else: + verbose_proxy_logger.warning( + f"WebSocket passthrough ({endpoint}): Failed to extract model from server setup response: {setup_response}" + ) + else: + verbose_proxy_logger.debug( + f"WebSocket passthrough ({endpoint}): Not a Vertex AI Live endpoint, skipping model extraction" + ) + + # Send the setup response to the client + await websocket.send_text(json.dumps(setup_response)) + + # Now continuously forward messages from upstream to client + async for upstream_message in upstream_ws: + if isinstance(upstream_message, bytes): + await websocket.send_bytes(upstream_message) + # Parse and collect for cost tracking + try: + message_data = json.loads(upstream_message.decode()) + websocket_messages.append(message_data) + except (json.JSONDecodeError, UnicodeDecodeError): + pass + else: + await websocket.send_text(upstream_message) + # Parse and collect for cost tracking + try: + message_data = json.loads(upstream_message) + websocket_messages.append(message_data) + except json.JSONDecodeError: + pass + + except (ConnectionClosedOK, ConnectionClosedError) as e: + verbose_proxy_logger.debug( + f"Upstream WebSocket connection closed: {e}" + ) + pass + except asyncio.CancelledError: + verbose_proxy_logger.debug( + "asyncio.CancelledError in forward_upstream_to_client" + ) + raise + except Exception as e: + verbose_proxy_logger.debug( + f"Exception in forward_upstream_to_client: {e}" + ) + verbose_proxy_logger.exception( + f"WebSocket passthrough ({endpoint}): error forwarding upstream message" + ) + raise + + # Create tasks for bidirectional message forwarding + tasks = [ + asyncio.create_task(forward_client_to_upstream()), + asyncio.create_task(forward_upstream_to_client()), + ] + + done, pending = await asyncio.wait( + tasks, return_when=asyncio.FIRST_COMPLETED + ) + + # Cancel remaining tasks + for task in pending: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + # Check for exceptions in completed tasks + for task in done: + exception = task.exception() + if exception is not None: + raise exception + + end_time = datetime.now() + + # Update passthrough logging payload with response data + passthrough_logging_payload["response_body"] = websocket_messages # type: ignore + passthrough_logging_payload["end_time"] = end_time # type: ignore + + # Remove logging_obj from kwargs to avoid duplicate keyword argument + success_kwargs = kwargs.copy() + success_kwargs.pop("logging_obj", None) + + # # Add user authentication context for database logging + # if user_api_key_dict: + # success_kwargs.setdefault('litellm_params', {}) + # success_kwargs['litellm_params'].update({ + # 'proxy_server_request': { + # 'body': { + # 'user': user_api_key_dict.user_id, + # 'team_id': user_api_key_dict.team_id, + # 'end_user_id': user_api_key_dict.end_user_id, + # } + # } + # }) + # # Also add the user_api_key for direct access + # success_kwargs['user_api_key'] = user_api_key_dict.api_key + + # Create a dummy httpx.Response for WebSocket connections + class MockWebSocketResponse: + def __init__(self, target_url: str): + self.status_code = 200 + self.text = "WebSocket connection successful" + self.headers: dict[str, str] = {} + self.request = MockWebSocketRequest(target_url) + + class MockWebSocketRequest: + def __init__(self, target_url: str): + self.method = "WEBSOCKET" + self.url = target_url + + mock_response = MockWebSocketResponse(target) + + # Use the same success handler as HTTP passthrough endpoints + asyncio.create_task( + pass_through_endpoint_logging.pass_through_async_success_handler( + httpx_response=mock_response, # type: ignore + response_body=websocket_messages, # type: ignore + url_route=endpoint or "", + result="websocket_connection_successful", + start_time=start_time, + end_time=end_time, + logging_obj=logging_obj, + cache_hit=False, + request_body={}, + **success_kwargs, + ) + ) + + # Call the proxy logging success hook + if proxy_logging_obj: + await proxy_logging_obj.post_call_success_hook( + data={}, + user_api_key_dict=user_api_key_dict, + response={"status": "websocket_connection_successful"}, # type: ignore + ) + + except InvalidStatus as exc: + verbose_proxy_logger.exception( + f"WebSocket passthrough ({endpoint}): upstream rejected WebSocket connection" + ) + + # Prepare request payload for logging + request_payload = {} + if kwargs: + for key, value in kwargs.items(): + request_payload[key] = value + if logging_obj is not None: + request_payload["litellm_logging_obj"] = logging_obj + + # Log the connection failure using the same pattern as HTTP + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=exc, + request_data=request_payload, + traceback_str=traceback.format_exc( + limit=MAXIMUM_TRACEBACK_LINES_TO_LOG, + ), + ) + + if websocket.client_state != WebSocketState.DISCONNECTED: + await websocket.close( + code=getattr(exc, "status_code", 1011), + reason="Upstream connection rejected", + ) + except Exception as e: + verbose_proxy_logger.exception( + f"WebSocket passthrough ({endpoint}): unexpected error while proxying WebSocket" + ) + + # Prepare request payload for logging + request_payload = {} + if kwargs: + for key, value in kwargs.items(): + request_payload[key] = value + if logging_obj is not None: + request_payload["litellm_logging_obj"] = logging_obj + + # Log the unexpected error using the same pattern as HTTP + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=e, + request_data=request_payload, + traceback_str=traceback.format_exc( + limit=MAXIMUM_TRACEBACK_LINES_TO_LOG, + ), + ) + + if websocket.client_state != WebSocketState.DISCONNECTED: + await websocket.close(code=1011, reason="WebSocket passthrough error") + finally: + if websocket.client_state != WebSocketState.DISCONNECTED: + await websocket.close() + + +def _is_streaming_response(response: httpx.Response) -> bool: + _content_type = response.headers.get("content-type") + if _content_type is not None and "text/event-stream" in _content_type: + return True + return False + + +def _extract_model_from_vertex_ai_setup(setup_response: dict) -> Optional[str]: + """ + Extract the model name from Vertex AI Live setup response. + + The setup response can contain a model field in two formats: + 1. Direct: {"model": "projects/.../models/gemini-2.0-flash-live-preview-04-09"} + 2. Nested: {"setup": {"model": "projects/.../models/gemini-2.0-flash-live-preview-04-09"}} + + We extract just the model name: "gemini-2.0-flash-live-preview-04-09" + """ + try: + # Handle both direct model field and nested setup.model field + model_path = None + if isinstance(setup_response, dict): + if "model" in setup_response: + model_path = setup_response["model"] + elif ( + "setup" in setup_response + and isinstance(setup_response["setup"], dict) + and "model" in setup_response["setup"] + ): + model_path = setup_response["setup"]["model"] + + if isinstance(model_path, str) and "/models/" in model_path: + # Extract the model name after the last "/models/" + model_name = model_path.split("/models/")[-1] + return model_name + except Exception as e: + verbose_proxy_logger.debug(f"Error extracting model from setup response: {e}") + return None + + +class SafeRouteAdder: + """ + Wrapper class for adding routes to FastAPI app. + Only adds routes if they don't already exist on the app. + """ + + @staticmethod + def _is_path_registered(app: FastAPI, path: str, methods: List[str]) -> bool: + """ + Check if a path with any of the specified methods is already registered on the app. + + Args: + app: The FastAPI application instance + path: The path to check (e.g., "/v1/chat/completions") + methods: List of HTTP methods to check (e.g., ["GET", "POST"]) + + Returns: + True if the path is already registered with any of the methods, False otherwise + """ + for route in app.routes: + # Use getattr to safely access route attributes + route_path = getattr(route, "path", None) + route_methods = getattr(route, "methods", None) + + if route_path == path and route_methods is not None: + # Check if any of the methods overlap + if any(method in route_methods for method in methods): + return True + return False + + @staticmethod + def add_api_route_if_not_exists( + app: FastAPI, + path: str, + endpoint: Any, + methods: List[str], + dependencies: Optional[List] = None, + ) -> bool: + """ + Add an API route to the app only if it doesn't already exist. + + Args: + app: The FastAPI application instance + path: The path for the route + endpoint: The endpoint function/callable + methods: List of HTTP methods + dependencies: Optional list of dependencies + + Returns: + True if route was added, False if it already existed + """ + if SafeRouteAdder._is_path_registered(app=app, path=path, methods=methods): + verbose_proxy_logger.debug( + "Skipping route registration - path %s with methods %s already registered on app", + path, + methods, + ) + return False + + app.add_api_route( + path=path, + endpoint=endpoint, + methods=methods, + dependencies=dependencies, + ) + verbose_proxy_logger.debug( + "Successfully added route: %s with methods %s", + path, + methods, + ) + return True + + +class InitPassThroughEndpointHelpers: + @staticmethod + def add_exact_path_route( + app: FastAPI, + path: str, + target: str, + custom_headers: Optional[dict], + forward_headers: Optional[bool], + merge_query_params: Optional[bool], + dependencies: Optional[List], + cost_per_request: Optional[float], + endpoint_id: str, + guardrails: Optional[dict] = None, + methods: Optional[List[str]] = None, + default_query_params: Optional[dict] = None, + config_file_path: Optional[str] = None, + ): + """Add exact path route for pass-through endpoint""" + # Default to all methods if none specified (backward compatibility) + if methods is None or len(methods) == 0: + methods = ["GET", "POST", "PUT", "DELETE", "PATCH"] + + # Create route key that includes methods for uniqueness + methods_str = ",".join(sorted(methods)) + route_key = f"{endpoint_id}:exact:{path}:{methods_str}" + + # Check if this exact route is already registered + if route_key in _registered_pass_through_routes: + verbose_proxy_logger.debug( + "Updating duplicate exact pass through endpoint: %s with methods %s (already registered)", + path, + methods, + ) + + verbose_proxy_logger.debug( + "adding exact pass through endpoint: %s, methods: %s, dependencies: %s", + path, + methods, + dependencies, + ) + + # Use SafeRouteAdder to only add route if it doesn't exist on the app + SafeRouteAdder.add_api_route_if_not_exists( + app=app, + path=path, + endpoint=create_pass_through_route( # type: ignore + path, + target, + custom_headers, + forward_headers, + merge_query_params, + dependencies, + cost_per_request=cost_per_request, + default_query_params=default_query_params, + guardrails=guardrails, + config_file_path=config_file_path, + ), + methods=methods, + dependencies=dependencies, + ) + + # Always register/update the route metadata (headers, target) even if FastAPI route exists + _registered_pass_through_routes[route_key] = { + "endpoint_id": endpoint_id, + "path": path, + "type": "exact", + "methods": methods, + "passthrough_params": { + "target": target, + "custom_headers": custom_headers, + "forward_headers": forward_headers, + "merge_query_params": merge_query_params, + "default_query_params": default_query_params, + "dependencies": dependencies, + "cost_per_request": cost_per_request, + "guardrails": guardrails, + }, + } + + @staticmethod + def add_subpath_route( + app: FastAPI, + path: str, + target: str, + custom_headers: Optional[dict], + forward_headers: Optional[bool], + merge_query_params: Optional[bool], + dependencies: Optional[List], + cost_per_request: Optional[float], + endpoint_id: str, + guardrails: Optional[dict] = None, + methods: Optional[List[str]] = None, + default_query_params: Optional[dict] = None, + config_file_path: Optional[str] = None, + ): + """Add wildcard route for sub-paths""" + # Default to all methods if none specified (backward compatibility) + if methods is None or len(methods) == 0: + methods = ["GET", "POST", "PUT", "DELETE", "PATCH"] + + wildcard_path = f"{path}/{{subpath:path}}" + methods_str = ",".join(sorted(methods)) + route_key = f"{endpoint_id}:subpath:{path}:{methods_str}" + + # Check if this subpath route is already registered + if route_key in _registered_pass_through_routes: + verbose_proxy_logger.debug( + "Updating duplicate wildcard pass through endpoint: %s with methods %s (already registered)", + wildcard_path, + methods, + ) + + verbose_proxy_logger.debug( + "adding wildcard pass through endpoint: %s, methods: %s, dependencies: %s", + wildcard_path, + methods, + dependencies, + ) + + # Use SafeRouteAdder to only add route if it doesn't exist on the app + SafeRouteAdder.add_api_route_if_not_exists( + app=app, + path=wildcard_path, + endpoint=create_pass_through_route( # type: ignore + path, + target, + custom_headers, + forward_headers, + merge_query_params, + dependencies, + include_subpath=True, + cost_per_request=cost_per_request, + default_query_params=default_query_params, + guardrails=guardrails, + config_file_path=config_file_path, + ), + methods=methods, + dependencies=dependencies, + ) + + # Register the route to prevent duplicates only if it was added + _registered_pass_through_routes[route_key] = { + "endpoint_id": endpoint_id, + "path": path, + "type": "subpath", + "methods": methods, + "passthrough_params": { + "target": target, + "custom_headers": custom_headers, + "forward_headers": forward_headers, + "merge_query_params": merge_query_params, + "default_query_params": default_query_params, + "dependencies": dependencies, + "cost_per_request": cost_per_request, + "guardrails": guardrails, + }, + } + + @staticmethod + def remove_endpoint_routes(endpoint_id: str): + """Remove all routes for a specific endpoint ID from the registry + and clean up corresponding entries from LiteLLMRoutes.openai_routes.""" + keys_to_remove = [ + key + for key, value in _registered_pass_through_routes.items() + if value["endpoint_id"] == endpoint_id + ] + for key in keys_to_remove: + route_info = _registered_pass_through_routes[key] + path = route_info.get("path") + if isinstance(path, str): + openai_routes = LiteLLMRoutes.openai_routes.value + if path in openai_routes: + openai_routes.remove(path) + if route_info.get("type") == "subpath": + wildcard_path = path.rstrip("/") + "/*" + if wildcard_path in openai_routes: + openai_routes.remove(wildcard_path) + del _registered_pass_through_routes[key] + verbose_proxy_logger.debug( + "Removed pass-through route from registry: %s", key + ) + + @staticmethod + def clear_all_pass_through_routes(): + """Clear all pass-through routes from the registry""" + _registered_pass_through_routes.clear() + + @staticmethod + def get_all_registered_pass_through_routes() -> List[str]: + """Get all registered pass-through endpoints from the registry""" + return list(_registered_pass_through_routes.keys()) + + @staticmethod + def _build_full_path_with_root(path: str) -> str: + """ + Build full path by prepending server root path if needed. + + Args: + path: The relative path to build + + Returns: + Full path with server root prepended (if root is not "/") + """ + root_path = get_server_root_path() + if root_path == "/": + return path + return f"{root_path}{path}" + + @staticmethod + def is_registered_pass_through_route(route: str) -> bool: + """ + Check if route is a registered pass-through endpoint from DB + + Uses the in-memory registry to avoid additional DB queries + Optimized for minimal latency + + Args: + route: The route to check + + Returns: + bool: True if route is a registered pass-through endpoint, False otherwise + """ + ## CHECK IF MAPPED PASS THROUGH ENDPOINT + normalized_route = normalize_route_for_root_path(route) + if normalized_route is not None: + for mapped_route in LiteLLMRoutes.mapped_pass_through_routes.value: + if normalized_route.startswith(mapped_route): + return True + + # Fast path: check if any registered route key contains this path + # Keys are in format: "{endpoint_id}:exact:{path}:{methods}" or "{endpoint_id}:subpath:{path}:{methods}" + # For backward compatibility, also support old format: "{endpoint_id}:exact:{path}" or "{endpoint_id}:subpath:{path}" + # Extract unique paths from keys for quick checking + for key in _registered_pass_through_routes.keys(): + parts = key.split(":", 3) # Split into [endpoint_id, type, path, methods?] + if len(parts) >= 3: + route_type = parts[1] + registered_path = ( + InitPassThroughEndpointHelpers._build_full_path_with_root(parts[2]) + ) + if route_type == "exact" and route == registered_path: + return True + elif route_type == "subpath": + if route == registered_path or route.startswith( + registered_path + "/" + ): + return True + + return False + + @staticmethod + def get_registered_pass_through_route( + route: str, method: Optional[str] = None + ) -> Optional[Dict[str, Any]]: + """Get passthrough params for a given route and optionally filter by HTTP method""" + for key in _registered_pass_through_routes.keys(): + parts = key.split(":", 3) # Split into [endpoint_id, type, path, methods?] + if len(parts) >= 3: + route_type = parts[1] + registered_path = ( + InitPassThroughEndpointHelpers._build_full_path_with_root(parts[2]) + ) + + # Get the methods for this route + route_methods = _registered_pass_through_routes[key].get("methods", []) + + # Check if path matches + path_matches = False + if route_type == "exact" and route == registered_path: + path_matches = True + elif route_type == "subpath": + if route == registered_path or route.startswith( + registered_path + "/" + ): + path_matches = True + + # If path matches and method filter is provided, check if method is allowed + if path_matches: + if method is None or not route_methods or method in route_methods: + return _registered_pass_through_routes[key] + + return None + + +def _get_combined_pass_through_endpoints( + pass_through_endpoints: Union[List[Dict], List[PassThroughGenericEndpoint]], + config_pass_through_endpoints: List[Dict], +): + """Get combined pass-through endpoints from db + config""" + return pass_through_endpoints + config_pass_through_endpoints + + +async def _register_pass_through_endpoint( + endpoint: Union[Dict[str, Any], PassThroughGenericEndpoint], + app: FastAPI, + premium_user: bool, + visited_endpoints: set[str], + config_file_path: Optional[str] = None, +) -> None: + endpoint_data: Dict[str, Any] + if isinstance(endpoint, PassThroughGenericEndpoint): + endpoint_data = endpoint.model_dump() + else: + endpoint_data = endpoint + + if endpoint_data.get("id") is None: + endpoint_data["id"] = str(uuid.uuid4()) + endpoint_id = cast(str, endpoint_data["id"]) + + target = endpoint_data.get("target") + path = endpoint_data.get("path") + if path is None: + raise ValueError("Path is required for pass-through endpoint") + + custom_headers = await set_env_variables_in_header( + custom_headers=endpoint_data.get("headers") + ) + forward_headers = endpoint_data.get("forward_headers") + merge_query_params = endpoint_data.get("merge_query_params") + default_query_params = endpoint_data.get("default_query_params") + auth = endpoint_data.get("auth") + dependencies = None + + if auth is not None and str(auth).lower() == "true": + # Authentication on a pass-through endpoint used to be enterprise-only. + # That left OSS with no safe configuration: auth=True raised at startup + # unless the operator had a license. The safe option must always be free, + # and unauthenticated forwarding should require explicit opt-in. + dependencies = [Depends(user_api_key_auth)] + if path not in LiteLLMRoutes.openai_routes.value: + LiteLLMRoutes.openai_routes.value.append(path) + + if target is None: + return + + guardrails = endpoint_data.get("guardrails") + methods = endpoint_data.get("methods") + cost_per_request = endpoint_data.get("cost_per_request") + + verbose_proxy_logger.debug( + "Initializing pass through endpoint: %s (ID: %s)", path, endpoint_id + ) + InitPassThroughEndpointHelpers.add_exact_path_route( + app=app, + path=path, + target=target, + custom_headers=custom_headers, + forward_headers=forward_headers, + merge_query_params=merge_query_params, + dependencies=dependencies, + cost_per_request=cost_per_request, + endpoint_id=endpoint_id, + guardrails=guardrails, + methods=methods, + default_query_params=default_query_params, + config_file_path=config_file_path, + ) + + methods_for_key = methods if methods else ["GET", "POST", "PUT", "DELETE", "PATCH"] + methods_str = ",".join(sorted(methods_for_key)) + visited_endpoints.add(f"{endpoint_id}:exact:{path}:{methods_str}") + + if endpoint_data.get("include_subpath", False) is True: + if auth is not None and str(auth).lower() == "true": + wildcard_path = path.rstrip("/") + "/*" + if wildcard_path not in LiteLLMRoutes.openai_routes.value: + LiteLLMRoutes.openai_routes.value.append(wildcard_path) + InitPassThroughEndpointHelpers.add_subpath_route( + app=app, + path=path, + target=target, + custom_headers=custom_headers, + forward_headers=forward_headers, + merge_query_params=merge_query_params, + dependencies=dependencies, + cost_per_request=cost_per_request, + endpoint_id=endpoint_id, + guardrails=guardrails, + methods=methods, + default_query_params=default_query_params, + config_file_path=config_file_path, + ) + visited_endpoints.add(f"{endpoint_id}:subpath:{path}:{methods_str}") + + verbose_proxy_logger.debug( + "Added new pass through endpoint: %s (ID: %s)", path, endpoint_id + ) + + +async def initialize_pass_through_endpoints( + pass_through_endpoints: Union[List[Dict], List[PassThroughGenericEndpoint]], + config_file_path: Optional[str] = None, +): + """ + 1. Create a global list of pass-through endpoints (db + config) + 2. Clear all existing pass-through endpoints from the FastAPI app routes + 3. Add new endpoints to the in-memory registry + + Initialize a list of pass-through endpoints by adding them to the FastAPI app routes + + Args: + pass_through_endpoints: List of pass-through endpoints to initialize + config_file_path: Path to the operator's config.yaml when this call + originates from a YAML-load. Threaded through to + ``create_pass_through_route`` so an operator using + ``s3://``/``gcs://`` ``custom_handler`` in their config still + loads. Callers from the DB-overlay / runtime API path must leave + this ``None`` so the runtime gate in ``get_instance_fn`` fires. + + Returns: + None + """ + verbose_proxy_logger.debug("initializing pass through endpoints") + from litellm.proxy.proxy_server import ( + app, + config_passthrough_endpoints, + premium_user, + ) + + ## get combined pass-through endpoints from db + config + combined_pass_through_endpoints: List[Union[Dict, PassThroughGenericEndpoint]] + + if config_passthrough_endpoints is not None: + combined_pass_through_endpoints = _get_combined_pass_through_endpoints( # type: ignore + pass_through_endpoints, config_passthrough_endpoints + ) + else: + combined_pass_through_endpoints = pass_through_endpoints # type: ignore + + ## clear all existing pass-through endpoints from the FastAPI app routes + # InitPassThroughEndpointHelpers.clear_all_pass_through_routes() + + # get a list of all registered pass-through endpoints + # mark the ones that are visited in the list + # remove the ones that are not visited from the list + registered_pass_through_endpoints = ( + InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() + ) + + visited_endpoints: set[str] = set() + + for endpoint in combined_pass_through_endpoints: + await _register_pass_through_endpoint( + endpoint=endpoint, + app=app, + premium_user=premium_user, + visited_endpoints=visited_endpoints, + config_file_path=config_file_path, + ) + + # remove the ones that are not visited from the list + for endpoint_key in registered_pass_through_endpoints: + if endpoint_key not in visited_endpoints: + InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_key) + + +def _get_pass_through_endpoints_from_config() -> List[PassThroughGenericEndpoint]: + """ + Get pass-through endpoints defined in the config file. + These are read-only and cannot be edited via the UI. + Malformed endpoints are logged and skipped; they do not crash the function. + """ + from pydantic import ValidationError + + from litellm.proxy.proxy_server import config_passthrough_endpoints + + if config_passthrough_endpoints is None or len(config_passthrough_endpoints) == 0: + return [] + + returned_endpoints: List[PassThroughGenericEndpoint] = [] + for endpoint in config_passthrough_endpoints: + try: + if isinstance(endpoint, dict): + endpoint_dict = dict(endpoint) + endpoint_dict["is_from_config"] = True + returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) + elif isinstance(endpoint, PassThroughGenericEndpoint): + # Create a copy with is_from_config=True + endpoint_dict = endpoint.model_dump() + endpoint_dict["is_from_config"] = True + returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) + except ValidationError as e: + verbose_proxy_logger.warning( + "Skipping malformed pass-through endpoint from config: %s", + e, + exc_info=False, + ) + + return returned_endpoints + + +async def _get_pass_through_endpoints_from_db( + endpoint_id: Optional[str] = None, + user_api_key_dict: Optional[UserAPIKeyAuth] = None, +) -> List[PassThroughGenericEndpoint]: + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.proxy_server import get_config_general_settings + + try: + if user_api_key_dict is None: + user_api_key_dict = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + response: ConfigFieldInfo = await get_config_general_settings( + field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict + ) + except Exception: + return [] + + pass_through_endpoint_data: Optional[List] = response.field_value + if pass_through_endpoint_data is None: + return [] + + returned_endpoints: List[PassThroughGenericEndpoint] = [] + if endpoint_id is None: + # Return all endpoints from DB, mark as not from config + for endpoint in pass_through_endpoint_data: + if isinstance(endpoint, dict): + endpoint_dict = dict(endpoint) + endpoint_dict["is_from_config"] = False + returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) + elif isinstance(endpoint, PassThroughGenericEndpoint): + endpoint_dict = endpoint.model_dump() + endpoint_dict["is_from_config"] = False + returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) + else: + # Find specific endpoint by ID + found_endpoint = _find_endpoint_by_id(pass_through_endpoint_data, endpoint_id) + if found_endpoint is not None: + endpoint_dict = ( + found_endpoint.model_dump() + if isinstance(found_endpoint, PassThroughGenericEndpoint) + else dict(found_endpoint) + ) + endpoint_dict["is_from_config"] = False + returned_endpoints.append(PassThroughGenericEndpoint(**endpoint_dict)) + + return returned_endpoints + + +async def _filter_endpoints_by_team_allowed_routes( + team_id: str, + pass_through_endpoints: List[PassThroughGenericEndpoint], + prisma_client, +) -> List[PassThroughGenericEndpoint]: + """ + Filter pass-through endpoints based on team's allowed_passthrough_routes metadata. + + Args: + team_id: The team ID to check permissions for + pass_through_endpoints: List of endpoints to filter + prisma_client: Database client + + Returns: + Filtered list of endpoints based on team permissions + + Raises: + HTTPException: If team is not found + """ + # retrieve team from db + team = await prisma_client.db.litellm_teamtable.find_unique( + where={"team_id": team_id}, + ) + if team is None: + raise HTTPException( + status_code=404, + detail={"error": "Team not found"}, + ) + + # retrieve team metadata + team_metadata = team.metadata + if ( + team_metadata is not None + and team_metadata.get("allowed_passthrough_routes") is not None + ): + ## FILTER pass_through_endpoints by allowed_passthrough_routes + pass_through_endpoints = [ + endpoint + for endpoint in pass_through_endpoints + if endpoint.path in team_metadata.get("allowed_passthrough_routes") + ] + + return pass_through_endpoints + + +@router.get( + "/config/pass_through_endpoint", + dependencies=[Depends(user_api_key_auth)], + response_model=PassThroughEndpointResponse, +) +@router.get( + "/config/pass_through_endpoint/team/{team_id}", + dependencies=[Depends(user_api_key_auth)], + response_model=PassThroughEndpointResponse, +) +async def get_pass_through_endpoints( + endpoint_id: Optional[str] = None, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + team_id: Optional[str] = None, +): + """ + GET configured pass through endpoint. + + If no endpoint_id given, return all configured endpoints. + """ ## Get existing pass-through endpoint field value + from litellm.proxy._types import CommonProxyErrors + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + + # Get endpoints from DB (editable via UI) + db_endpoints = await _get_pass_through_endpoints_from_db( + endpoint_id=endpoint_id, user_api_key_dict=user_api_key_dict + ) + + # Get endpoints from config file (read-only, not editable via UI) + config_endpoints = _get_pass_through_endpoints_from_config() + + # Merge: config endpoints not in DB + all DB endpoints (DB overrides config for same path) + db_paths = {ep.path for ep in db_endpoints} + config_only_endpoints = [ep for ep in config_endpoints if ep.path not in db_paths] + if endpoint_id is not None: + # When filtering by endpoint_id, only return if found in DB (config endpoints use generated IDs) + pass_through_endpoints = db_endpoints + else: + pass_through_endpoints = config_only_endpoints + db_endpoints + + if team_id is not None: + pass_through_endpoints = await _filter_endpoints_by_team_allowed_routes( + team_id=team_id, + pass_through_endpoints=pass_through_endpoints, + prisma_client=prisma_client, + ) + + return PassThroughEndpointResponse(endpoints=pass_through_endpoints) + + +@router.post( + "/config/pass_through_endpoint/{endpoint_id}", + dependencies=[Depends(user_api_key_auth)], +) +async def update_pass_through_endpoints( + endpoint_id: str, + data: PassThroughGenericEndpoint, + request: Request, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Update a pass-through endpoint by ID. + """ + from litellm.proxy.proxy_server import ( + get_config_general_settings, + update_config_general_settings, + ) + + ## Get existing pass-through endpoint field value + try: + response: ConfigFieldInfo = await get_config_general_settings( + field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict + ) + except Exception: + raise HTTPException( + status_code=404, + detail={"error": "No pass-through endpoints found"}, + ) + + pass_through_endpoint_data: Optional[List] = response.field_value + if pass_through_endpoint_data is None: + raise HTTPException( + status_code=404, + detail={"error": "No pass-through endpoints found"}, + ) + + # Find the endpoint to update + found_endpoint = _find_endpoint_by_id(pass_through_endpoint_data, endpoint_id) + + if found_endpoint is None: + raise HTTPException( + status_code=404, + detail={"error": f"Endpoint with ID '{endpoint_id}' not found"}, + ) + + # Find the index for updating the list + endpoint_index = None + for idx, endpoint in enumerate(pass_through_endpoint_data): + _endpoint = ( + PassThroughGenericEndpoint(**endpoint) + if isinstance(endpoint, dict) + else endpoint + ) + if _endpoint.id == endpoint_id: + endpoint_index = idx + break + + if endpoint_index is None: + raise HTTPException( + status_code=404, + detail={ + "error": f"Could not find index for endpoint with ID '{endpoint_id}'" + }, + ) + + # Get the update data as dict, excluding None values for partial updates + # Exclude is_from_config as it's a response-only field (computed at read time) + update_data = data.model_dump(exclude_none=True, exclude={"is_from_config"}) + + # Start with existing endpoint data + endpoint_dict = found_endpoint.model_dump() + + # Update with new data (only non-None values) + endpoint_dict.update(update_data) + + # Preserve existing ID if not provided in update and endpoint has ID + if "id" not in update_data and found_endpoint.id is not None: + endpoint_dict["id"] = found_endpoint.id + + # Remove is_from_config before saving - it's a response-only field (computed at read time) + endpoint_dict.pop("is_from_config", None) + + # Create updated endpoint object + updated_endpoint = PassThroughGenericEndpoint(**endpoint_dict) + + # Update the list + pass_through_endpoint_data[endpoint_index] = endpoint_dict + + # Remove old routes from registry before they get re-registered + InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_id) + + ## Update db + updated_data = ConfigFieldUpdate( + field_name="pass_through_endpoints", + field_value=pass_through_endpoint_data, + config_type="general_settings", + ) + + await update_config_general_settings( + data=updated_data, user_api_key_dict=user_api_key_dict + ) + + # Re-register the route with updated headers + _custom_headers: Optional[dict] = updated_endpoint.headers or {} + _custom_headers = await set_env_variables_in_header(custom_headers=_custom_headers) + + if updated_endpoint.include_subpath: + InitPassThroughEndpointHelpers.add_subpath_route( + app=request.app, + path=updated_endpoint.path, + target=updated_endpoint.target, + custom_headers=_custom_headers, + forward_headers=None, # Defaults not available in model? assuming None logic handles it + merge_query_params=None, + dependencies=None, + cost_per_request=updated_endpoint.cost_per_request, + endpoint_id=updated_endpoint.id or endpoint_id or "", + guardrails=getattr(updated_endpoint, "guardrails", None), + methods=updated_endpoint.methods, + default_query_params=updated_endpoint.default_query_params, + ) + else: + InitPassThroughEndpointHelpers.add_exact_path_route( + app=request.app, + path=updated_endpoint.path, + target=updated_endpoint.target, + custom_headers=_custom_headers, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=updated_endpoint.cost_per_request, + endpoint_id=updated_endpoint.id or endpoint_id or "", + guardrails=getattr(updated_endpoint, "guardrails", None), + methods=updated_endpoint.methods, + default_query_params=updated_endpoint.default_query_params, + ) + + return PassThroughEndpointResponse( + endpoints=[updated_endpoint] if updated_endpoint else [] + ) + + +@router.post( + "/config/pass_through_endpoint", + dependencies=[Depends(user_api_key_auth)], +) +async def create_pass_through_endpoints( + data: PassThroughGenericEndpoint, + request: Request, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Create new pass-through endpoint + """ + from litellm._uuid import uuid + from litellm.proxy.proxy_server import ( + get_config_general_settings, + update_config_general_settings, + ) + + ## Get existing pass-through endpoint field value + + try: + response: ConfigFieldInfo = await get_config_general_settings( + field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict + ) + except Exception: + response = ConfigFieldInfo( + field_name="pass_through_endpoints", field_value=None + ) + + ## Auto-generate ID if not provided + # Exclude is_from_config as it's a response-only field (computed at read time) + data_dict = data.model_dump(exclude={"is_from_config"}) + if data_dict.get("id") is None: + data_dict["id"] = str(uuid.uuid4()) + + if response.field_value is None: + response.field_value = [data_dict] + elif isinstance(response.field_value, List): + response.field_value.append(data_dict) + + ## Update db + updated_data = ConfigFieldUpdate( + field_name="pass_through_endpoints", + field_value=response.field_value, + config_type="general_settings", + ) + await update_config_general_settings( + data=updated_data, user_api_key_dict=user_api_key_dict + ) + + # Return the created endpoint with the generated ID + created_endpoint = PassThroughGenericEndpoint(**data_dict) + + # Register the new route + _custom_headers: Optional[dict] = created_endpoint.headers or {} + _custom_headers = await set_env_variables_in_header(custom_headers=_custom_headers) + + if created_endpoint.include_subpath: + InitPassThroughEndpointHelpers.add_subpath_route( + app=request.app, + path=created_endpoint.path, + target=created_endpoint.target, + custom_headers=_custom_headers, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=created_endpoint.cost_per_request, + endpoint_id=created_endpoint.id or "", + guardrails=getattr(created_endpoint, "guardrails", None), + methods=created_endpoint.methods, + default_query_params=created_endpoint.default_query_params, + ) + else: + InitPassThroughEndpointHelpers.add_exact_path_route( + app=request.app, + path=created_endpoint.path, + target=created_endpoint.target, + custom_headers=_custom_headers, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=created_endpoint.cost_per_request, + endpoint_id=created_endpoint.id or "", + guardrails=getattr(created_endpoint, "guardrails", None), + methods=created_endpoint.methods, + default_query_params=created_endpoint.default_query_params, + ) + + return PassThroughEndpointResponse(endpoints=[created_endpoint]) + + +@router.delete( + "/config/pass_through_endpoint", + dependencies=[Depends(user_api_key_auth)], + response_model=PassThroughEndpointResponse, +) +async def delete_pass_through_endpoints( + endpoint_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Delete a pass-through endpoint by ID. + + Returns - the deleted endpoint + """ + from litellm.proxy.proxy_server import ( + get_config_general_settings, + update_config_general_settings, + ) + + ## Get existing pass-through endpoint field value + + try: + response: ConfigFieldInfo = await get_config_general_settings( + field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict + ) + except Exception: + response = ConfigFieldInfo( + field_name="pass_through_endpoints", field_value=None + ) + + ## Update field by removing endpoint + pass_through_endpoint_data: Optional[List] = response.field_value + if response.field_value is None or pass_through_endpoint_data is None: + raise HTTPException( + status_code=400, + detail={"error": "There are no pass-through endpoints setup."}, + ) + + # Find the endpoint to delete + found_endpoint = _find_endpoint_by_id(pass_through_endpoint_data, endpoint_id) + + if found_endpoint is None: + raise HTTPException( + status_code=400, + detail={ + "error": "Endpoint with ID '{}' was not found in pass-through endpoint list.".format( + endpoint_id + ) + }, + ) + + # Find the index for deleting from the list + endpoint_index = None + for idx, endpoint in enumerate(pass_through_endpoint_data): + _endpoint = ( + PassThroughGenericEndpoint(**endpoint) + if isinstance(endpoint, dict) + else endpoint + ) + if _endpoint.id == endpoint_id: + endpoint_index = idx + break + + if endpoint_index is None: + raise HTTPException( + status_code=400, + detail={ + "error": f"Could not find index for endpoint with ID '{endpoint_id}'" + }, + ) + + # Remove the endpoint + pass_through_endpoint_data.pop(endpoint_index) + response_obj = found_endpoint + + # Remove routes from registry + InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_id) + + ## Update db + updated_data = ConfigFieldUpdate( + field_name="pass_through_endpoints", + field_value=pass_through_endpoint_data, + config_type="general_settings", + ) + await update_config_general_settings( + data=updated_data, user_api_key_dict=user_api_key_dict + ) + + return PassThroughEndpointResponse(endpoints=[response_obj]) + + +def _find_endpoint_by_id( + endpoints_data: List, + endpoint_id: str, +) -> Optional[PassThroughGenericEndpoint]: + """ + Find an endpoint by ID. + + Args: + endpoints_data: List of endpoint data (dicts or PassThroughGenericEndpoint objects) + endpoint_id: ID to search for + + Returns: + Found endpoint or None if not found + """ + for endpoint in endpoints_data: + _endpoint: Optional[PassThroughGenericEndpoint] = None + if isinstance(endpoint, dict): + _endpoint = PassThroughGenericEndpoint(**endpoint) + elif isinstance(endpoint, PassThroughGenericEndpoint): + _endpoint = endpoint + + # Only compare IDs to IDs + if _endpoint is not None and _endpoint.id == endpoint_id: + return _endpoint + + return None + + +async def initialize_pass_through_endpoints_in_db(): + """ + Gets all pass-through endpoints from db and initializes them in the proxy server. + """ + pass_through_endpoints = await _get_pass_through_endpoints_from_db() + await initialize_pass_through_endpoints( + pass_through_endpoints=pass_through_endpoints + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2a08507743..b296792cd0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15644,7 +15644,6 @@ async def get_routes(): app.include_router(router) app.include_router(response_router) -app.include_router(batches_router) app.include_router(public_endpoints_router) app.include_router(rerank_router) app.include_router(ocr_router) @@ -15657,6 +15656,7 @@ app.include_router(fine_tuning_router) app.include_router(credential_router) app.include_router(llm_passthrough_router) app.include_router(pass_through_router) +app.include_router(batches_router) app.include_router(health_router) app.include_router(key_management_router) app.include_router(internal_user_router) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 8f471b62b5..5574d616fa 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3602,6 +3602,10 @@ class SpecialEnums(Enum): "litellm:custom_llm_provider:{};model_id:{};video_id:{}" ) + LITELLM_PASSTHROUGH_MANAGED_ID_COMPLETE_STR = ( + "litellm_proxy:passthrough;provider:{};unified_id,{};raw_id,{}" + ) + class ServiceTier(Enum): """Enum for service tier types used in cost calculations.""" diff --git a/tests/pass_through_unit_tests/test_passthrough_managed_ids.py b/tests/pass_through_unit_tests/test_passthrough_managed_ids.py new file mode 100644 index 0000000000..8cf07da3ce --- /dev/null +++ b/tests/pass_through_unit_tests/test_passthrough_managed_ids.py @@ -0,0 +1,2087 @@ +""" +Unit tests for passthrough managed IDs (Scope A). + +Tests cover: + - managed_id_codec: encode / decode / is_managed round-trip and rejection cases. + - managed_id_rewriter._resolve_one: cross-route 404, access-check 403, unknown ID 404, + raw pass-through. + - managed_id_rewriter.rewrite_response_ids: file create swap, batch create swap, + dedup reuse (no duplicate row), null field skip. + - managed_id_rewriter.rewrite_path_ids / rewrite_query_ids / rewrite_body_ids: + INPUT swap and raw pass-through. + - Flag-off: feature flag disabled → no swap at all. + - Cross-route: managed ID minted for 'openai' rejected on a different provider. + - Forged: unknown base64 → 404. +""" + +from __future__ import annotations + +import base64 +import json +import sys +import os +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm.llms.base_llm.managed_resources.utils import ( + resolve_passthrough_managed_id_provider, +) +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.pass_through_endpoints.managed_id_codec import ( + decode, + encode, + is_managed, + new_managed_id, +) +from litellm.proxy.pass_through_endpoints.managed_id_rewriter import ( + _MAX_RAW_ID_GUARD_LOOKUPS, + _canonical_path, + _passthrough_provider_marker, + _resolve_one, + is_passthrough_list_route, + list_passthrough_ids_from_db, + rewrite_body_ids, + rewrite_path_ids, + rewrite_query_ids, + rewrite_response_ids, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _user(user_id: str = "user-1", team_id: str = "team-1") -> UserAPIKeyAuth: + return UserAPIKeyAuth(user_id=user_id, team_id=team_id) + + +def _admin_user() -> UserAPIKeyAuth: + u = UserAPIKeyAuth(user_id="admin", user_role="proxy_admin") + return u + + +def _prisma_client() -> MagicMock: + """Return a MagicMock prisma_client with async db methods.""" + pc = MagicMock() + pc.db = MagicMock() + pc.db.litellm_managedfiletable = MagicMock() + pc.db.litellm_managedfiletable.find_first = AsyncMock(return_value=None) + pc.db.litellm_managedfiletable.find_many = AsyncMock(return_value=[]) + pc.db.litellm_managedfiletable.create = AsyncMock(return_value=None) + pc.db.litellm_managedobjecttable = MagicMock() + pc.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=None) + pc.db.litellm_managedobjecttable.upsert = AsyncMock(return_value=None) + pc.db.litellm_managedobjecttable.update = AsyncMock(return_value=None) + return pc + + +def _managed_files_hook(store_side_effect: Any = None) -> MagicMock: + hook = MagicMock() + hook.get_unified_file_id = AsyncMock(return_value=None) + hook.store_unified_file_id = AsyncMock(side_effect=store_side_effect) + return hook + + +def _owner_scoped_file_find_many(row: Any): + """Return a ``find_many`` that mimics Prisma owner-scoping for the managed + file table: an owner-scoped query (one carrying ``created_by`` / ``team_id`` + / ``OR``) returns ``[]`` because the caller does not own *row*, while an + unscoped (global) query returns ``[row]``. This reproduces the cross-tenant + bypass that a caller-scoped dedup lookup allowed (the scoped query misses the + other tenant's row, so a fresh managed ID gets minted for the attacker).""" + + async def _impl(*args: Any, where: Any = None, **kwargs: Any) -> Any: + where = where or {} + if "created_by" in where or "team_id" in where or "OR" in where: + return [] + return [row] + + return _impl + + +# --------------------------------------------------------------------------- +# managed_id_codec — unit tests +# --------------------------------------------------------------------------- + + +class TestCodec: + def test_encode_decode_roundtrip(self): + managed_id = encode("openai", "uuid-abc", "file-xyz") + payload = decode(managed_id) + assert payload is not None + assert payload.provider == "openai" + assert payload.unified_uuid == "uuid-abc" + assert payload.raw_provider_id == "file-xyz" + + def test_is_managed_true(self): + assert is_managed(encode("openai", "u1", "file-abc")) is True + + def test_is_managed_false_for_raw_ids(self): + assert is_managed("file-abc123") is False + assert is_managed("batch_xyz") is False + assert is_managed("resp_abc") is False + + def test_decode_returns_none_for_garbage(self): + assert decode("not-base64!!!") is None + assert decode("") is None + assert decode("abc") is None + + def test_decode_returns_none_for_wrong_type(self): + assert decode(None) is None # type: ignore[arg-type] + assert decode(42) is None # type: ignore[arg-type] + + def test_decode_returns_none_for_unified_endpoint_id(self): + # A unified-endpoint ID: starts with litellm_proxy: but lacks passthrough; + plaintext = "litellm_proxy:application/octet-stream;unified_id,123;target_model_names,gpt-4" + unified_id = base64.urlsafe_b64encode(plaintext.encode()).decode().rstrip("=") + assert decode(unified_id) is None + + def test_new_managed_id_produces_valid_id(self): + mid = new_managed_id("openai", "batch_abc") + payload = decode(mid) + assert payload is not None + assert payload.provider == "openai" + assert payload.raw_provider_id == "batch_abc" + + def test_encode_padding_insensitive(self): + """Encoded IDs with varying lengths all decode correctly.""" + for raw in ("file-x", "file-ab", "file-abc", "file-abcd"): + mid = encode("openai", "u", raw) + p = decode(mid) + assert p is not None and p.raw_provider_id == raw + + +# --------------------------------------------------------------------------- +# resolve_passthrough_managed_id_provider — provider scope mapping +# --------------------------------------------------------------------------- + + +class TestManagedIdProviderScope: + """Managed-ID scoping is keyed on the explicit forwarded provider, and both + azure and azure_ai must collapse to a single 'azure' scope so an ID minted + while routing as one resolves while routing as the other.""" + + def test_openai_scope(self): + assert resolve_passthrough_managed_id_provider("openai") == "openai" + assert ( + resolve_passthrough_managed_id_provider(litellm.LlmProviders.OPENAI) + == "openai" + ) + + def test_azure_scope(self): + assert resolve_passthrough_managed_id_provider("azure") == "azure" + assert ( + resolve_passthrough_managed_id_provider(litellm.LlmProviders.AZURE) + == "azure" + ) + + def test_azure_ai_collapses_to_azure(self): + assert resolve_passthrough_managed_id_provider("azure_ai") == "azure" + assert ( + resolve_passthrough_managed_id_provider(litellm.LlmProviders.AZURE_AI) + == "azure" + ) + + def test_azure_ai_id_resolves_on_azure_route(self): + """End-to-end consequence of the collapse: an ID whose scope was + resolved from azure_ai shares the 'azure' namespace, so decoding + + cross-route checks line up with an azure-scoped ID.""" + azure_ai_scope = resolve_passthrough_managed_id_provider("azure_ai") + azure_scope = resolve_passthrough_managed_id_provider("azure") + managed = new_managed_id(azure_ai_scope, "file-shared") + assert decode(managed).provider == azure_scope + + def test_case_insensitive(self): + assert resolve_passthrough_managed_id_provider("AZURE") == "azure" + assert resolve_passthrough_managed_id_provider("OpenAI") == "openai" + + def test_namespaced_provider_suffix(self): + assert resolve_passthrough_managed_id_provider("foo.azure") == "azure" + assert resolve_passthrough_managed_id_provider("foo.azure_ai") == "azure" + assert resolve_passthrough_managed_id_provider("foo.openai") == "openai" + + def test_non_openai_azure_providers_not_scoped(self): + """Managed IDs only apply to explicit openai/azure pass-through; any + other provider (or a missing one) must return None so a third-party + OpenAI-compatible endpoint never triggers managed-ID minting.""" + for provider in (None, "", "cohere", "vllm", "anthropic", "gemini", "bedrock"): + assert resolve_passthrough_managed_id_provider(provider) is None + + +# --------------------------------------------------------------------------- +# _canonical_path +# --------------------------------------------------------------------------- + + +class TestCanonicalPath: + def test_strips_openai_prefix(self): + assert _canonical_path("/openai/v1/batches/batch_x") == "/v1/batches/batch_x" + + def test_strips_openai_passthrough_prefix(self): + assert _canonical_path("/openai_passthrough/v1/files") == "/v1/files" + + def test_leaves_bare_path_unchanged(self): + assert _canonical_path("/v1/responses") == "/v1/responses" + + def test_strips_azure_openai_prefix(self): + assert _canonical_path("/azure/openai/files") == "/v1/files" + + def test_strips_azure_openai_batch_with_id(self): + assert ( + _canonical_path("/azure/openai/batches/batch_abc123") + == "/v1/batches/batch_abc123" + ) + + def test_strips_azure_openai_responses(self): + assert _canonical_path("/azure/openai/responses") == "/v1/responses" + + def test_strips_azure_ai_openai_prefix(self): + assert _canonical_path("/azure_ai/openai/files") == "/v1/files" + + def test_strips_azure_ai_openai_batch_cancel(self): + assert ( + _canonical_path("/azure_ai/openai/batches/batch_abc/cancel") + == "/v1/batches/batch_abc/cancel" + ) + + def test_azure_path_already_carrying_v1_is_not_doubled(self): + assert _canonical_path("/azure/openai/v1/files") == "/v1/files" + assert ( + _canonical_path("/azure/openai/v1/batches/batch_abc") + == "/v1/batches/batch_abc" + ) + + def test_strips_azure_openai_file_with_id(self): + assert _canonical_path("/azure/openai/files/file-abc") == "/v1/files/file-abc" + + +# --------------------------------------------------------------------------- +# _resolve_one +# --------------------------------------------------------------------------- + + +class TestResolveOne: + @pytest.mark.asyncio + async def test_raw_id_passes_through(self): + result = await _resolve_one("file-abc", "openai", _user(), None, None) + assert result == "file-abc" + + @pytest.mark.asyncio + async def test_cross_route_raises_404(self): + mid = encode("anthropic", "u", "file-abc") + from fastapi import HTTPException + + with pytest.raises(HTTPException) as exc_info: + await _resolve_one(mid, "openai", _user(), None, None) + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_unknown_managed_id_raises_404(self): + mid = encode("openai", "u", "file-abc") + pc = _prisma_client() + hook = _managed_files_hook() + # Both lookups return None → 404 + from fastapi import HTTPException + + with pytest.raises(HTTPException) as exc_info: + await _resolve_one(mid, "openai", _user(), pc, hook) + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_access_denied_raises_403(self): + mid = encode("openai", "u", "file-abc") + hook = _managed_files_hook() + file_row = MagicMock() + file_row.created_by = "other-user" + file_row.team_id = "other-team" + hook.get_unified_file_id = AsyncMock(return_value=file_row) + from fastapi import HTTPException + + with pytest.raises(HTTPException) as exc_info: + await _resolve_one(mid, "openai", _user("user-1", "team-1"), None, hook) + assert exc_info.value.status_code == 403 + + @pytest.mark.asyncio + async def test_valid_file_id_resolves(self): + mid = encode("openai", "u", "file-xyz") + hook = _managed_files_hook() + file_row = MagicMock() + file_row.created_by = "user-1" + file_row.team_id = "team-1" + hook.get_unified_file_id = AsyncMock(return_value=file_row) + result = await _resolve_one(mid, "openai", _user(), None, hook) + assert result == "file-xyz" + + @pytest.mark.asyncio + async def test_valid_batch_id_resolves_via_object_table(self): + mid = encode("openai", "u", "batch_abc") + pc = _prisma_client() + obj_row = MagicMock() + obj_row.created_by = "user-1" + obj_row.team_id = "team-1" + pc.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=obj_row) + result = await _resolve_one(mid, "openai", _user(), pc, None) + assert result == "batch_abc" + + @pytest.mark.asyncio + async def test_admin_can_access_any_resource(self): + mid = encode("openai", "u", "file-xyz") + hook = _managed_files_hook() + file_row = MagicMock() + file_row.created_by = "other-user" + file_row.team_id = "other-team" + hook.get_unified_file_id = AsyncMock(return_value=file_row) + result = await _resolve_one(mid, "openai", _admin_user(), None, hook) + assert result == "file-xyz" + + +# --------------------------------------------------------------------------- +# rewrite_response_ids — OUTPUT +# --------------------------------------------------------------------------- + + +class TestRewriteResponseIds: + @pytest.mark.asyncio + async def test_file_create_mints_managed_id(self): + pc = _prisma_client() + hook = _managed_files_hook() + body = {"id": "file-abc123", "object": "file"} + result = await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai/v1/files", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + assert result is not body # mutated copy + assert result["id"] != "file-abc123" + payload = decode(result["id"]) + assert payload is not None + assert payload.raw_provider_id == "file-abc123" + hook.store_unified_file_id.assert_awaited_once() + + @pytest.mark.asyncio + async def test_file_create_persist_failure_leaves_raw_id(self): + """If the DB write fails, the response must keep the raw provider ID + (which still resolves upstream) rather than swap in a managed ID that no + DB row backs and that would 404 on every later resolve.""" + pc = _prisma_client() + hook = _managed_files_hook(store_side_effect=Exception("db down")) + body = {"id": "file-abc123", "object": "file"} + result = await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai/v1/files", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + hook.store_unified_file_id.assert_awaited_once() + assert result["id"] == "file-abc123" + assert decode(result["id"]) is None + + @pytest.mark.asyncio + async def test_batch_create_mints_id_and_input_file_id(self): + pc = _prisma_client() + hook = _managed_files_hook() + body = { + "id": "batch_xyz", + "input_file_id": "file-abc", + "output_file_id": None, + "error_file_id": None, + } + result = await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai/v1/batches", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + assert decode(result["id"]).raw_provider_id == "batch_xyz" # type: ignore[union-attr] + assert decode(result["input_file_id"]).raw_provider_id == "file-abc" # type: ignore[union-attr] + # Null fields skipped + assert result["output_file_id"] is None + assert result["error_file_id"] is None + + @pytest.mark.asyncio + async def test_response_create_mints_id(self): + pc = _prisma_client() + hook = _managed_files_hook() + body = {"id": "resp_abc", "object": "response"} + result = await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai/v1/responses", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + assert decode(result["id"]).raw_provider_id == "resp_abc" # type: ignore[union-attr] + + @pytest.mark.asyncio + async def test_azure_response_create_mints_id(self): + pc = _prisma_client() + hook = _managed_files_hook() + body = { + "id": "resp_0dce2668af072bdc006a195db1f96c8194b6217f8e0d0b3ccd", + "object": "response", + "status": "completed", + } + result = await rewrite_response_ids( + provider="azure", + method="POST", + route="/azure/openai/responses", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + assert ( + decode(result["id"]).raw_provider_id # type: ignore[union-attr] + == "resp_0dce2668af072bdc006a195db1f96c8194b6217f8e0d0b3ccd" + ) + + @pytest.mark.asyncio + async def test_no_map_entry_returns_body_unchanged(self): + pc = _prisma_client() + hook = _managed_files_hook() + body = {"id": "msg_xyz", "object": "message"} + result = await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai/v1/chat/completions", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + assert result is body # same object, unchanged + + @pytest.mark.asyncio + async def test_dedup_reuses_existing_file_row(self): + """File uploaded via passthrough, then referenced in a batch — no new row.""" + existing_managed_id = new_managed_id("openai", "file-abc") + existing_row = MagicMock() + existing_row.unified_file_id = existing_managed_id + existing_row.created_by = "user-1" + existing_row.team_id = "team-1" + + pc = _prisma_client() + # Dedup lookup finds existing row + pc.db.litellm_managedfiletable.find_many = AsyncMock( + return_value=[existing_row] + ) + hook = _managed_files_hook() + body = { + "id": "batch_xyz", + "input_file_id": "file-abc", + "output_file_id": None, + "error_file_id": None, + } + result = await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai/v1/batches", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + # input_file_id should be the SAME managed ID already in DB + assert result["input_file_id"] == existing_managed_id + # store_unified_file_id should NOT have been called (reused existing) + hook.store_unified_file_id.assert_not_awaited() + + @pytest.mark.asyncio + async def test_dedup_skips_cross_provider_file_row(self): + """Same raw file ID for a different provider must mint a new managed ID.""" + azure_managed_id = new_managed_id("azure", "file-abc") + existing_row = MagicMock() + existing_row.unified_file_id = azure_managed_id + + pc = _prisma_client() + pc.db.litellm_managedfiletable.find_many = AsyncMock( + return_value=[existing_row] + ) + hook = _managed_files_hook() + body = {"id": "file-abc", "object": "file"} + result = await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai/v1/files", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + assert decode(result["id"]).provider == "openai" + assert decode(result["id"]).raw_provider_id == "file-abc" + assert result["id"] != azure_managed_id + hook.store_unified_file_id.assert_awaited_once() + + @pytest.mark.asyncio + async def test_dedup_reuses_same_provider_row_amid_collision(self): + """When OpenAI and Azure both issued the same raw file ID, an Azure call + must reuse the existing Azure managed row deterministically rather than + mint a duplicate, even when the cross-provider OpenAI row is returned + first by the DB.""" + raw_id = "file-collision" + openai_row = MagicMock() + openai_row.unified_file_id = new_managed_id("openai", raw_id) + openai_row.created_by = "user-1" + openai_row.team_id = "team-1" + azure_managed_id = new_managed_id("azure", raw_id) + azure_row = MagicMock() + azure_row.unified_file_id = azure_managed_id + azure_row.created_by = "user-1" + azure_row.team_id = "team-1" + + pc = _prisma_client() + # Cross-provider row listed first to expose any non-deterministic pick. + pc.db.litellm_managedfiletable.find_many = AsyncMock( + return_value=[openai_row, azure_row] + ) + hook = _managed_files_hook() + body = {"id": raw_id, "object": "file"} + result = await rewrite_response_ids( + provider="azure", + method="GET", + route=f"/azure/openai/files/{raw_id}", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + assert result["id"] == azure_managed_id + hook.store_unified_file_id.assert_not_awaited() + + @pytest.mark.asyncio + async def test_cross_owner_file_retrieve_raises_404(self): + """ + A caller who fetches another tenant's raw ``file-...`` ID through + GET /openai/v1/files/{file_id} (which bypasses the managed-ID input gate) + must be denied with a 404 — the response path must NOT mint a fresh + managed ID for that file under the attacker. + """ + from fastapi import HTTPException + + pc = _prisma_client() + other_owner_row = MagicMock() + other_owner_row.created_by = "victim" + other_owner_row.team_id = "victim-team" + other_owner_row.unified_file_id = encode("openai", "victim", "file-victim") + pc.db.litellm_managedfiletable.find_many = _owner_scoped_file_find_many( + other_owner_row + ) + hook = _managed_files_hook() + + body = {"id": "file-victim", "object": "file"} + with pytest.raises(HTTPException) as exc_info: + await rewrite_response_ids( + provider="openai", + method="GET", + route="/openai/v1/files/file-victim", + body=body, + user_api_key_dict=_user("attacker", "attacker-team"), + prisma_client=pc, + managed_files_hook=hook, + ) + assert exc_info.value.status_code == 404 + # Must not mint / persist a managed ID for the attacker. + hook.store_unified_file_id.assert_not_awaited() + + @pytest.mark.asyncio + async def test_cross_owner_file_delete_raises_404(self): + """DELETE is also a non-create route: cross-owner raw file IDs are denied.""" + from fastapi import HTTPException + + pc = _prisma_client() + other_owner_row = MagicMock() + other_owner_row.created_by = "victim" + other_owner_row.team_id = "victim-team" + other_owner_row.unified_file_id = encode("openai", "victim", "file-victim") + pc.db.litellm_managedfiletable.find_many = _owner_scoped_file_find_many( + other_owner_row + ) + hook = _managed_files_hook() + + body = {"id": "file-victim", "object": "file", "deleted": True} + with pytest.raises(HTTPException) as exc_info: + await rewrite_response_ids( + provider="openai", + method="DELETE", + route="/openai/v1/files/file-victim", + body=body, + user_api_key_dict=_user("attacker", "attacker-team"), + prisma_client=pc, + managed_files_hook=hook, + ) + assert exc_info.value.status_code == 404 + hook.store_unified_file_id.assert_not_awaited() + + @pytest.mark.asyncio + async def test_cross_owner_file_create_leaves_raw_id(self): + """ + On the create (POST /v1/files) path a cross-owner dedup hit must NOT 404 + the caller's own successful upload; leave the raw ID unmanaged instead + (mirrors the batch/response create behaviour). + """ + pc = _prisma_client() + other_owner_row = MagicMock() + other_owner_row.created_by = "victim" + other_owner_row.team_id = "victim-team" + other_owner_row.unified_file_id = encode("openai", "victim", "file-shared") + pc.db.litellm_managedfiletable.find_many = _owner_scoped_file_find_many( + other_owner_row + ) + hook = _managed_files_hook() + + body = {"id": "file-shared", "object": "file"} + result = await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai/v1/files", + body=body, + user_api_key_dict=_user("uploader", "uploader-team"), + prisma_client=pc, + managed_files_hook=hook, + ) + assert result["id"] == "file-shared" + hook.store_unified_file_id.assert_not_awaited() + + @pytest.mark.asyncio + async def test_team_member_reuses_shared_file_row(self): + """A teammate of the file owner can reuse the existing managed file row + (the cross-tenant guard scopes by team, not just the creating user).""" + existing_managed_id = new_managed_id("openai", "file-team") + existing_row = MagicMock() + existing_row.unified_file_id = existing_managed_id + existing_row.created_by = "owner-user" + existing_row.team_id = "shared-team" + + pc = _prisma_client() + pc.db.litellm_managedfiletable.find_many = AsyncMock( + return_value=[existing_row] + ) + hook = _managed_files_hook() + + body = {"id": "file-team", "object": "file"} + result = await rewrite_response_ids( + provider="openai", + method="GET", + route="/openai/v1/files/file-team", + body=body, + user_api_key_dict=_user("teammate", "shared-team"), + prisma_client=pc, + managed_files_hook=hook, + ) + assert result["id"] == existing_managed_id + hook.store_unified_file_id.assert_not_awaited() + + @pytest.mark.asyncio + async def test_openai_passthrough_prefix_normalised(self): + """Routes under /openai_passthrough/ work the same as /openai/.""" + pc = _prisma_client() + hook = _managed_files_hook() + body = {"id": "file-abc", "object": "file"} + result = await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai_passthrough/v1/files", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + assert decode(result["id"]).raw_provider_id == "file-abc" # type: ignore[union-attr] + + @pytest.mark.asyncio + async def test_batch_reuse_refreshes_stored_snapshot(self): + """Retrieving a completed batch must refresh the stored snapshot so the + DB-served list reflects fields (e.g. output_file_id) that were null at + creation time. The dedup-reuse path must update file_object, not just + return the existing id with a stale snapshot.""" + existing_managed_id = new_managed_id("openai", "batch_done") + existing_row = MagicMock() + existing_row.unified_object_id = existing_managed_id + existing_row.created_by = "user-1" + existing_row.team_id = "team-1" + + pc = _prisma_client() + pc.db.litellm_managedobjecttable.find_first = AsyncMock( + return_value=existing_row + ) + + completed_body = { + "id": "batch_done", + "object": "batch", + "status": "completed", + "output_file_id": "file-out", + "error_file_id": None, + } + result = await rewrite_response_ids( + provider="openai", + method="GET", + route="/openai/v1/batches/batch_done", + body=completed_body, + user_api_key_dict=_user("user-1", "team-1"), + prisma_client=pc, + managed_files_hook=None, + ) + + # Reuses the existing managed id (no new row minted) + assert result["id"] == existing_managed_id + pc.db.litellm_managedobjecttable.upsert.assert_not_awaited() + # The stored snapshot is refreshed with the completed batch body + pc.db.litellm_managedobjecttable.update.assert_awaited_once() + update_kwargs = pc.db.litellm_managedobjecttable.update.call_args.kwargs + assert update_kwargs["where"] == {"unified_object_id": existing_managed_id} + stored = json.loads(update_kwargs["data"]["file_object"]) + assert stored["status"] == "completed" + # output_file_id is itself rewritten to a managed id wrapping the raw id + assert decode(stored["output_file_id"]).raw_provider_id == "file-out" + + @pytest.mark.asyncio + async def test_cross_provider_batch_collision_mints_new_id(self): + """ + If OpenAI and Azure independently issue the same raw batch ID, the + Azure call must mint its own row keyed by 'passthrough:azure:batch_shared' + and must NOT raise 404. The namespaced model_object_id prevents a + UniqueConstraintViolation on the @unique column. + """ + pc = _prisma_client() + # Both providers return no existing row (different namespaced keys) + pc.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=None) + pc.db.litellm_managedobjecttable.upsert = AsyncMock(return_value=None) + + body = {"id": "batch_shared", "object": "batch", "input_file_id": None} + result = await rewrite_response_ids( + provider="azure", + method="POST", + route="/azure/openai/batches", + body=body, + user_api_key_dict=_user("user-azure", "team-azure"), + prisma_client=pc, + managed_files_hook=None, + ) + # Must mint a fresh azure-scoped managed ID + assert decode(result["id"]) is not None + assert decode(result["id"]).provider == "azure" + assert decode(result["id"]).raw_provider_id == "batch_shared" + + # Verify the upsert stored the namespaced model_object_id + call_data = pc.db.litellm_managedobjecttable.upsert.call_args.kwargs["data"] + assert ( + call_data["create"]["model_object_id"] == "passthrough:azure:batch_shared" + ) + + @pytest.mark.asyncio + async def test_batch_create_persist_failure_leaves_raw_id(self): + """If the object upsert fails, the batch response must keep the raw + provider ID rather than return a managed ID with no backing DB row that + would 404 on every subsequent resolve.""" + pc = _prisma_client() + pc.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=None) + pc.db.litellm_managedobjecttable.upsert = AsyncMock( + side_effect=Exception("db down") + ) + body = {"id": "batch_xyz", "object": "batch", "input_file_id": None} + result = await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai/v1/batches", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=None, + ) + pc.db.litellm_managedobjecttable.upsert.assert_awaited_once() + assert result["id"] == "batch_xyz" + assert decode(result["id"]) is None + + @pytest.mark.asyncio + async def test_concurrent_create_converges_on_winner_managed_id(self): + """ + Two callers minting the same namespaced object row race: the dedup lookup + finds nothing for both, but the @unique model_object_id lets only one + insert win. The loser's upsert raises, and it must re-read the winner's + row and return that managed ID rather than silently keeping the raw ID + (which would leave the two callers divergent for the same upstream batch). + """ + pc = _prisma_client() + winner_managed_id = encode("openai", "winner-uuid", "batch_race") + winner_row = MagicMock() + winner_row.created_by = "user-1" + winner_row.team_id = "team-1" + winner_row.unified_object_id = winner_managed_id + # First (dedup) lookup misses; post-collision re-read finds the winner. + pc.db.litellm_managedobjecttable.find_first = AsyncMock( + side_effect=[None, winner_row] + ) + pc.db.litellm_managedobjecttable.upsert = AsyncMock( + side_effect=Exception("UniqueConstraintViolation: model_object_id") + ) + + body = {"id": "batch_race", "object": "batch", "input_file_id": None} + result = await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai/v1/batches", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=None, + ) + # The loser converges on the winner's managed ID, not the raw batch ID. + assert result["id"] == winner_managed_id + assert decode(result["id"]).raw_provider_id == "batch_race" + assert pc.db.litellm_managedobjecttable.find_first.await_count == 2 + + @pytest.mark.asyncio + async def test_concurrent_create_race_with_cross_owner_winner_retrieve_404(self): + """ + If the row that wins the insert race on a non-create (retrieve) route is + owned by a different tenant, the loser must be denied with 404 rather + than handed the raw ID — the post-collision re-read runs the same access + check as the initial dedup hit. + """ + from fastapi import HTTPException + + pc = _prisma_client() + winner_row = MagicMock() + winner_row.created_by = "other-user" + winner_row.team_id = "other-team" + winner_row.unified_object_id = encode("openai", "other-uuid", "batch_race") + pc.db.litellm_managedobjecttable.find_first = AsyncMock( + side_effect=[None, winner_row] + ) + pc.db.litellm_managedobjecttable.upsert = AsyncMock( + side_effect=Exception("UniqueConstraintViolation: model_object_id") + ) + + body = {"id": "batch_race", "object": "batch", "input_file_id": None} + with pytest.raises(HTTPException) as exc_info: + await rewrite_response_ids( + provider="openai", + method="GET", + route="/openai/v1/batches/batch_race", + body=body, + user_api_key_dict=_user("attacker", "attacker-team"), + prisma_client=pc, + managed_files_hook=None, + ) + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_cross_provider_batch_collision_dedup_uses_namespaced_key(self): + """ + When OpenAI already has a row for batch_shared, an Azure request must + look up 'passthrough:azure:batch_shared' (not 'batch_shared'), find + nothing, and mint a new row — not raise 404 or reuse the OpenAI row. + """ + pc = _prisma_client() + # Simulate: OpenAI row exists under 'passthrough:openai:batch_shared', + # but Azure lookup for 'passthrough:azure:batch_shared' returns None. + pc.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=None) + pc.db.litellm_managedobjecttable.upsert = AsyncMock(return_value=None) + + body = {"id": "batch_shared", "object": "batch", "input_file_id": None} + result = await rewrite_response_ids( + provider="azure", + method="POST", + route="/azure/openai/batches", + body=body, + user_api_key_dict=_user("user-azure", "team-azure"), + prisma_client=pc, + managed_files_hook=None, + ) + # The dedup lookup must use the namespaced key + lookup_where = pc.db.litellm_managedobjecttable.find_first.call_args.kwargs[ + "where" + ] + assert lookup_where["model_object_id"] == "passthrough:azure:batch_shared" + # Result is a valid azure-scoped managed ID + assert decode(result["id"]).provider == "azure" + + @pytest.mark.asyncio + async def test_cross_owner_object_collision_returns_raw_id_not_404(self): + """ + On the OUTPUT (mint) path, if the namespaced key is already owned by a + different caller (e.g. two upstream accounts under one provider name + issued the same raw batch ID), the caller's successful upstream create + must NOT be turned into a 404. Leave their raw ID unmanaged instead. + """ + pc = _prisma_client() + other_owner_row = MagicMock() + other_owner_row.created_by = "other-user" + other_owner_row.team_id = "other-team" + other_owner_row.unified_object_id = encode( + "azure", "other-user", "batch_shared" + ) + pc.db.litellm_managedobjecttable.find_first = AsyncMock( + return_value=other_owner_row + ) + pc.db.litellm_managedobjecttable.upsert = AsyncMock(return_value=None) + + body = {"id": "batch_shared", "object": "batch", "input_file_id": None} + result = await rewrite_response_ids( + provider="azure", + method="POST", + route="/azure/openai/batches", + body=body, + user_api_key_dict=_user("user-azure", "team-azure"), + prisma_client=pc, + managed_files_hook=None, + ) + # Caller gets their raw batch ID back, unmanaged; not a 404, and not + # the other owner's managed ID. + assert result["id"] == "batch_shared" + # No new row is minted (would violate the @unique model_object_id). + pc.db.litellm_managedobjecttable.upsert.assert_not_awaited() + + @pytest.mark.asyncio + async def test_cross_owner_object_retrieve_raises_404(self): + """ + On a retrieve route, a caller who supplies another owner's raw batch ID + (which bypasses the managed-ID input gate) must be denied with a 404 — + the upstream object must NOT be echoed back with its raw ID. + """ + from fastapi import HTTPException + + pc = _prisma_client() + other_owner_row = MagicMock() + other_owner_row.created_by = "other-user" + other_owner_row.team_id = "other-team" + other_owner_row.unified_object_id = encode("openai", "other-user", "batch_xyz") + pc.db.litellm_managedobjecttable.find_first = AsyncMock( + return_value=other_owner_row + ) + + body = {"id": "batch_xyz", "object": "batch", "input_file_id": None} + with pytest.raises(HTTPException) as exc_info: + await rewrite_response_ids( + provider="openai", + method="GET", + route="/openai/v1/batches/batch_xyz", + body=body, + user_api_key_dict=_user("attacker", "attacker-team"), + prisma_client=pc, + managed_files_hook=None, + ) + assert exc_info.value.status_code == 404 + # Must not silently mint a row for the attacker either. + pc.db.litellm_managedobjecttable.upsert.assert_not_awaited() + + @pytest.mark.asyncio + async def test_cross_owner_response_delete_raises_404(self): + """A delete route is also a non-create route: cross-owner access is denied.""" + from fastapi import HTTPException + + pc = _prisma_client() + other_owner_row = MagicMock() + other_owner_row.created_by = "other-user" + other_owner_row.team_id = "other-team" + other_owner_row.unified_object_id = encode("openai", "other-user", "resp_abc") + pc.db.litellm_managedobjecttable.find_first = AsyncMock( + return_value=other_owner_row + ) + + body = {"id": "resp_abc", "object": "response"} + with pytest.raises(HTTPException) as exc_info: + await rewrite_response_ids( + provider="openai", + method="DELETE", + route="/openai/v1/responses/resp_abc", + body=body, + user_api_key_dict=_user("attacker", "attacker-team"), + prisma_client=pc, + managed_files_hook=None, + ) + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_batch_retrieve_swaps_output_file_id(self): + pc = _prisma_client() + hook = _managed_files_hook() + body = { + "id": "batch_xyz", + "input_file_id": "file-in", + "output_file_id": "file-out", + "error_file_id": "file-err", + } + result = await rewrite_response_ids( + provider="openai", + method="GET", + route="/openai/v1/batches/batch_xyz", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + assert decode(result["output_file_id"]).raw_provider_id == "file-out" # type: ignore[union-attr] + assert decode(result["error_file_id"]).raw_provider_id == "file-err" # type: ignore[union-attr] + + @pytest.mark.asyncio + async def test_file_create_persists_metadata_for_list(self): + """The file's upstream metadata is stored so the DB-served list returns + the same fields as a direct file GET (managed ID swapped in).""" + pc = _prisma_client() + hook = _managed_files_hook() + body = { + "id": "file-abc123", + "object": "file", + "bytes": 120, + "created_at": 1234567890, + "filename": "train.jsonl", + "purpose": "batch", + "status": "processed", + } + result = await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai/v1/files", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + stored = hook.store_unified_file_id.call_args.kwargs["file_object"] + assert stored is not None + assert stored.filename == "train.jsonl" + assert stored.bytes == 120 + assert stored.purpose == "batch" + # Managed ID is swapped into the persisted metadata (never the raw one). + assert stored.id == result["id"] + assert decode(stored.id).raw_provider_id == "file-abc123" # type: ignore[union-attr] + + @pytest.mark.asyncio + async def test_file_create_without_metadata_stores_no_file_object(self): + """A minimal file response (no bytes/filename) falls back to storing the + row without metadata rather than raising.""" + pc = _prisma_client() + hook = _managed_files_hook() + body = {"id": "file-abc123", "object": "file"} + await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai/v1/files", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + hook.store_unified_file_id.assert_awaited_once() + assert hook.store_unified_file_id.call_args.kwargs["file_object"] is None + + @pytest.mark.asyncio + async def test_file_create_persists_provider_marker_for_list_scope(self): + """The minted file row must carry the provider marker (it flows into + flat_model_file_ids), or the DB-pushed provider scope in + list_passthrough_ids_from_db would never match it.""" + pc = _prisma_client() + hook = _managed_files_hook() + await rewrite_response_ids( + provider="azure", + method="POST", + route="/azure/openai/files", + body={"id": "file-abc123", "object": "file"}, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + mappings = hook.store_unified_file_id.call_args.kwargs["model_mappings"] + assert _passthrough_provider_marker("azure") in mappings.values() + assert _passthrough_provider_marker("openai") not in mappings.values() + + @pytest.mark.asyncio + async def test_batch_snapshot_stores_managed_nested_file_ids(self): + """The persisted batch snapshot must carry the managed nested file ID so + the list response matches the rewritten direct GET response.""" + import json as _json + + pc = _prisma_client() + hook = _managed_files_hook() + body = { + "id": "batch_xyz", + "object": "batch", + "input_file_id": "file-in", + "output_file_id": None, + "error_file_id": None, + } + result = await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai/v1/batches", + body=body, + user_api_key_dict=_user(), + prisma_client=pc, + managed_files_hook=hook, + ) + stored = pc.db.litellm_managedobjecttable.upsert.call_args.kwargs["data"][ + "create" + ]["file_object"] + snapshot = _json.loads(stored) + assert snapshot["input_file_id"] == result["input_file_id"] + assert decode(snapshot["input_file_id"]).raw_provider_id == "file-in" # type: ignore[union-attr] + + +# --------------------------------------------------------------------------- +# rewrite_path_ids — INPUT +# --------------------------------------------------------------------------- + + +class TestRewritePathIds: + @pytest.mark.asyncio + async def test_raw_segment_passes_through(self): + result = await rewrite_path_ids( + "/v1/batches/batch_abc", "openai", _user(), None, None + ) + assert result == "/v1/batches/batch_abc" + + @pytest.mark.asyncio + async def test_managed_segment_is_resolved(self): + mid = encode("openai", "u", "batch_abc") + hook = _managed_files_hook() + pc = _prisma_client() + obj_row = MagicMock() + obj_row.created_by = "user-1" + obj_row.team_id = "team-1" + pc.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=obj_row) + result = await rewrite_path_ids( + f"/v1/batches/{mid}", "openai", _user(), pc, hook + ) + assert result == "/v1/batches/batch_abc" + + @pytest.mark.asyncio + async def test_cross_route_in_path_raises_404(self): + mid = encode("anthropic", "u", "batch_abc") + from fastapi import HTTPException + + with pytest.raises(HTTPException) as exc_info: + await rewrite_path_ids(f"/v1/batches/{mid}", "openai", _user(), None, None) + assert exc_info.value.status_code == 404 + + +# --------------------------------------------------------------------------- +# rewrite_query_ids — INPUT +# --------------------------------------------------------------------------- + + +class TestRewriteQueryIds: + @pytest.mark.asyncio + async def test_raw_params_pass_through(self): + params = {"limit": "10", "after": "batch_xyz"} + result = await rewrite_query_ids(params, "openai", _user(), None, None) + assert result is params # unchanged same object + + @pytest.mark.asyncio + async def test_none_returns_none(self): + result = await rewrite_query_ids(None, "openai", _user(), None, None) + assert result is None + + @pytest.mark.asyncio + async def test_managed_param_is_resolved(self): + mid = encode("openai", "u", "file-abc") + hook = _managed_files_hook() + file_row = MagicMock() + file_row.created_by = "user-1" + file_row.team_id = "team-1" + hook.get_unified_file_id = AsyncMock(return_value=file_row) + params = {"file_id": mid} + result = await rewrite_query_ids(params, "openai", _user(), None, hook) + assert result is not params + assert result["file_id"] == "file-abc" # type: ignore[index] + + +# --------------------------------------------------------------------------- +# rewrite_body_ids — INPUT +# --------------------------------------------------------------------------- + + +class TestRewriteBodyIds: + @pytest.mark.asyncio + async def test_raw_body_passes_through(self): + body = {"input_file_id": "file-abc", "model": "gpt-4o"} + result = await rewrite_body_ids(body, "openai", _user(), None, None) + assert result is body + + @pytest.mark.asyncio + async def test_none_returns_none(self): + result = await rewrite_body_ids(None, "openai", _user(), None, None) + assert result is None + + @pytest.mark.asyncio + async def test_managed_id_in_body_resolved(self): + mid = encode("openai", "u", "file-xyz") + hook = _managed_files_hook() + file_row = MagicMock() + file_row.created_by = "user-1" + file_row.team_id = "team-1" + hook.get_unified_file_id = AsyncMock(return_value=file_row) + body = {"input_file_id": mid} + result = await rewrite_body_ids(body, "openai", _user(), None, hook) + assert result is not body + assert result["input_file_id"] == "file-xyz" # type: ignore[index] + + @pytest.mark.asyncio + async def test_litellm_internal_key_preserved(self): + """litellm_logging_obj and similar keys are never walked.""" + logging_obj = object() + body = {"litellm_logging_obj": logging_obj, "model": "gpt-4o"} + result = await rewrite_body_ids(body, "openai", _user(), None, None) + # Internal key preserved by reference + assert result["litellm_logging_obj"] is logging_obj # type: ignore[index] + + @pytest.mark.asyncio + async def test_nested_list_resolved(self): + """Managed IDs inside nested lists are resolved.""" + mid = encode("openai", "u", "file-nested") + hook = _managed_files_hook() + file_row = MagicMock() + file_row.created_by = "user-1" + file_row.team_id = "team-1" + hook.get_unified_file_id = AsyncMock(return_value=file_row) + body = {"files": [mid, "raw-string"]} + result = await rewrite_body_ids(body, "openai", _user(), None, hook) + assert result["files"][0] == "file-nested" # type: ignore[index] + assert result["files"][1] == "raw-string" # type: ignore[index] + + @pytest.mark.asyncio + async def test_forged_managed_id_raises_404(self): + """An unknown managed ID in the body raises 404 (not passed to upstream).""" + mid = encode("openai", "u", "file-forged") + hook = _managed_files_hook() + hook.get_unified_file_id = AsyncMock(return_value=None) + pc = _prisma_client() + body = {"input_file_id": mid} + from fastapi import HTTPException + + with pytest.raises(HTTPException) as exc_info: + await rewrite_body_ids(body, "openai", _user(), pc, hook) + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_cross_user_access_denied_in_body(self): + """A managed ID owned by a different user raises 403.""" + mid = encode("openai", "u", "file-other") + hook = _managed_files_hook() + file_row = MagicMock() + file_row.created_by = "other-user" + file_row.team_id = "other-team" + hook.get_unified_file_id = AsyncMock(return_value=file_row) + body = {"input_file_id": mid} + from fastapi import HTTPException + + with pytest.raises(HTTPException) as exc_info: + await rewrite_body_ids( + body, "openai", _user("user-1", "team-1"), None, hook + ) + assert exc_info.value.status_code == 403 + + @pytest.mark.asyncio + async def test_deeply_nested_body_does_not_overflow_stack(self): + """A pathologically deep body must not blow the Python stack: rewriting + stops at the depth cap and returns the body unchanged instead of raising + RecursionError.""" + node: Any = {"leaf": "raw-value"} + for _ in range(5000): + node = {"nested": node} + + result = await rewrite_body_ids(node, "openai", _user(), None, None) + assert result is node + + @pytest.mark.asyncio + async def test_managed_id_resolved_within_depth_cap(self): + """A managed ID nested well within the depth cap is still resolved, so + the cap never truncates legitimately-shaped bodies.""" + mid = encode("openai", "u", "file-deep") + hook = _managed_files_hook() + file_row = MagicMock() + file_row.created_by = "user-1" + file_row.team_id = "team-1" + hook.get_unified_file_id = AsyncMock(return_value=file_row) + + leaf = {"input_file_id": mid} + node: Any = leaf + for _ in range(20): + node = {"nested": node} + + result = await rewrite_body_ids(node, "openai", _user(), None, hook) + + cursor = result + for _ in range(20): + cursor = cursor["nested"] # type: ignore[index] + assert cursor["input_file_id"] == "file-deep" # type: ignore[index] + + +# --------------------------------------------------------------------------- +# Raw-provider-ID input guard — a raw ID recovered by decoding another tenant's +# managed ID must NOT be forwarded upstream when it maps to a managed resource +# the caller does not own (otherwise a DELETE / cancel runs upstream before the +# response-side ownership check). +# --------------------------------------------------------------------------- + + +class TestRawProviderIdInputGuard: + @staticmethod + def _victim_file_row() -> MagicMock: + row = MagicMock() + row.created_by = "victim" + row.team_id = "victim-team" + row.unified_file_id = encode("openai", "victim", "file-victim") + return row + + @staticmethod + def _victim_object_row() -> MagicMock: + row = MagicMock() + row.created_by = "victim" + row.team_id = "victim-team" + row.unified_object_id = encode("openai", "victim", "batch_victim") + return row + + @pytest.mark.asyncio + async def test_raw_file_path_for_other_owner_denied(self): + """DELETE /openai/v1/files/file-victim with a raw ID that belongs to + another tenant's managed file is rejected (404) before forwarding.""" + from fastapi import HTTPException + + pc = _prisma_client() + pc.db.litellm_managedfiletable.find_many = AsyncMock( + return_value=[self._victim_file_row()] + ) + with pytest.raises(HTTPException) as exc_info: + await rewrite_path_ids( + "/openai/v1/files/file-victim", + "openai", + _user("attacker", "attacker-team"), + pc, + _managed_files_hook(), + ) + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_raw_batch_cancel_path_for_other_owner_denied(self): + """POST /openai/v1/batches/batch_victim/cancel with another tenant's raw + batch ID is rejected (404) before the upstream cancel runs.""" + from fastapi import HTTPException + + pc = _prisma_client() + pc.db.litellm_managedobjecttable.find_first = AsyncMock( + return_value=self._victim_object_row() + ) + with pytest.raises(HTTPException) as exc_info: + await rewrite_path_ids( + "/openai/v1/batches/batch_victim/cancel", + "openai", + _user("attacker", "attacker-team"), + pc, + _managed_files_hook(), + ) + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_raw_file_query_for_other_owner_denied(self): + from fastapi import HTTPException + + pc = _prisma_client() + pc.db.litellm_managedfiletable.find_many = AsyncMock( + return_value=[self._victim_file_row()] + ) + with pytest.raises(HTTPException) as exc_info: + await rewrite_query_ids( + {"file_id": "file-victim"}, + "openai", + _user("attacker", "attacker-team"), + pc, + _managed_files_hook(), + ) + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_raw_file_body_for_other_owner_denied(self): + from fastapi import HTTPException + + pc = _prisma_client() + pc.db.litellm_managedfiletable.find_many = AsyncMock( + return_value=[self._victim_file_row()] + ) + with pytest.raises(HTTPException) as exc_info: + await rewrite_body_ids( + {"input_file_id": "file-victim"}, + "openai", + _user("attacker", "attacker-team"), + pc, + _managed_files_hook(), + ) + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_raw_file_owned_by_caller_passes_through(self): + """A raw ID the caller does own is left untouched and forwarded — the + guard must not block legitimate raw-ID usage.""" + pc = _prisma_client() + own_row = MagicMock() + own_row.created_by = "user-1" + own_row.team_id = "team-1" + own_row.unified_file_id = encode("openai", "u", "file-mine") + pc.db.litellm_managedfiletable.find_many = AsyncMock(return_value=[own_row]) + result = await rewrite_path_ids( + "/openai/v1/files/file-mine", + "openai", + _user("user-1", "team-1"), + pc, + _managed_files_hook(), + ) + assert result == "/openai/v1/files/file-mine" + + @pytest.mark.asyncio + async def test_unmanaged_raw_id_passes_through(self): + """A raw ID with no managed row at all is a genuine opt-out and is + forwarded unchanged.""" + pc = _prisma_client() + result = await rewrite_path_ids( + "/openai/v1/files/file-never-managed", + "openai", + _user("attacker", "attacker-team"), + pc, + _managed_files_hook(), + ) + assert result == "/openai/v1/files/file-never-managed" + + @pytest.mark.asyncio + async def test_cross_provider_raw_file_not_blocked(self): + """A raw ID whose only managed row belongs to a different provider is not + this provider's resource, so the guard does not deny it.""" + pc = _prisma_client() + azure_row = MagicMock() + azure_row.created_by = "victim" + azure_row.team_id = "victim-team" + azure_row.unified_file_id = encode("azure", "victim", "file-victim") + pc.db.litellm_managedfiletable.find_many = AsyncMock(return_value=[azure_row]) + result = await rewrite_path_ids( + "/openai/v1/files/file-victim", + "openai", + _user("attacker", "attacker-team"), + pc, + _managed_files_hook(), + ) + assert result == "/openai/v1/files/file-victim" + + +# --------------------------------------------------------------------------- +# Raw-provider-ID guard amplification — a body packed with id-shaped strings +# must not fan out into one (unindexed) DB scan per string. The guard de-dupes +# repeats and caps the distinct lookups per request, failing closed instead of +# skipping the guard. +# --------------------------------------------------------------------------- + + +class TestRawProviderIdGuardBudget: + @pytest.mark.asyncio + async def test_many_distinct_raw_ids_capped(self): + """A body with more distinct raw file IDs than the per-request budget is + rejected with 400, and the number of (unindexed) DB scans never exceeds + the cap.""" + from fastapi import HTTPException + + pc = _prisma_client() + body = {"ids": [f"file-{i}" for i in range(_MAX_RAW_ID_GUARD_LOOKUPS + 25)]} + with pytest.raises(HTTPException) as exc_info: + await rewrite_body_ids( + body, "openai", _user("attacker", "attacker-team"), pc, None + ) + assert exc_info.value.status_code == 400 + assert ( + pc.db.litellm_managedfiletable.find_many.call_count + == _MAX_RAW_ID_GUARD_LOOKUPS + ) + + @pytest.mark.asyncio + async def test_repeated_raw_id_deduped(self): + """The same raw ID repeated many times issues exactly one DB lookup.""" + pc = _prisma_client() + body = {"ids": ["file-dup"] * (_MAX_RAW_ID_GUARD_LOOKUPS * 5)} + result = await rewrite_body_ids( + body, "openai", _user("attacker", "attacker-team"), pc, None + ) + assert result is body + assert pc.db.litellm_managedfiletable.find_many.call_count == 1 + + @pytest.mark.asyncio + async def test_distinct_ids_under_cap_not_rejected(self): + """A realistically-sized body (few distinct raw IDs) is never rejected and + each distinct ID is guarded once.""" + pc = _prisma_client() + body = {"ids": [f"file-{i}" for i in range(5)]} + result = await rewrite_body_ids( + body, "openai", _user("user-1", "team-1"), pc, None + ) + assert result is body + assert pc.db.litellm_managedfiletable.find_many.call_count == 5 + + @pytest.mark.asyncio + async def test_budget_is_per_input_surface(self): + """Each input surface (path / query / body) gets its own budget, so a + request distributing IDs across them is still bounded per surface.""" + from fastapi import HTTPException + + pc = _prisma_client() + params = {f"k{i}": f"file-{i}" for i in range(_MAX_RAW_ID_GUARD_LOOKUPS + 5)} + with pytest.raises(HTTPException) as exc_info: + await rewrite_query_ids( + params, "openai", _user("attacker", "attacker-team"), pc, None + ) + assert exc_info.value.status_code == 400 + assert ( + pc.db.litellm_managedfiletable.find_many.call_count + == _MAX_RAW_ID_GUARD_LOOKUPS + ) + + +# --------------------------------------------------------------------------- +# Flag-off: behaviour unchanged when passthrough_managed_object_ids is False +# --------------------------------------------------------------------------- + + +class TestFlagOff: + """ + When the feature flag is off the pass_through_request code paths skip both + hooks entirely. Here we verify the rewriter modules themselves are pure + no-ops when called with no DB / hook: raw IDs pass through. + """ + + @pytest.mark.asyncio + async def test_raw_file_in_response_not_swapped_without_hook(self): + body = {"id": "file-abc", "object": "file"} + result = await rewrite_response_ids( + provider="openai", + method="POST", + route="/openai/v1/files", + body=body, + user_api_key_dict=_user(), + prisma_client=None, + managed_files_hook=None, + ) + # Without DB/hook, _mint_or_reuse_file returns raw_id unchanged + assert result is body or result["id"] == "file-abc" + + @pytest.mark.asyncio + async def test_decode_failure_body_untouched(self): + body = {"id": "file-abc123"} + result = await rewrite_body_ids(body, "openai", _user(), None, None) + assert result is body + + +# --------------------------------------------------------------------------- +# list_passthrough_ids_from_db — unit tests +# --------------------------------------------------------------------------- + + +def _prisma_with_list(file_rows=None, batch_rows=None) -> MagicMock: + """Return a prisma_client whose find_many honors the provider scope pushed + into the ``where`` clause, mirroring how Postgres would filter rows. + + File rows are scoped via ``flat_model_file_ids: {has: }`` and object + rows via ``model_object_id: {startswith: passthrough::}``; the mock + applies the same predicate so a test feeding mixed-provider rows exercises + the real DB-pushdown contract instead of an unscoped passthrough.""" + pc = _prisma_client() + + def _file_filter(*args, where=None, take=None, **kwargs): + rows = list(file_rows or []) + marker = (where or {}).get("flat_model_file_ids", {}) or {} + marker = marker.get("has") + if marker is not None: + rows = [ + r + for r in rows + if marker in (getattr(r, "flat_model_file_ids", None) or []) + ] + return rows if take is None else rows[:take] + + def _batch_filter(*args, where=None, take=None, **kwargs): + rows = list(batch_rows or []) + prefix = (where or {}).get("model_object_id", {}) or {} + prefix = prefix.get("startswith") + if prefix is not None: + rows = [ + r + for r in rows + if str(getattr(r, "model_object_id", "") or "").startswith(prefix) + ] + return rows if take is None else rows[:take] + + if file_rows is not None: + pc.db.litellm_managedfiletable.find_many = AsyncMock(side_effect=_file_filter) + if batch_rows is not None: + pc.db.litellm_managedobjecttable.find_many = AsyncMock( + side_effect=_batch_filter + ) + return pc + + +def _fake_file_row( + unified_id: str, created_by: str = "user-1", team_id: str = "team-1" +): + row = MagicMock() + row.unified_file_id = unified_id + row.created_by = created_by + row.team_id = team_id + row.file_object = {"filename": "test.jsonl", "bytes": 42, "purpose": "batch"} + payload = decode(unified_id) + row.flat_model_file_ids = ( + [payload.raw_provider_id, _passthrough_provider_marker(payload.provider)] + if payload is not None + else [] + ) + + import datetime + + row.created_at = datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc) + return row + + +def _fake_batch_row( + unified_id: str, created_by: str = "user-1", team_id: str = "team-1" +): + row = MagicMock() + row.unified_object_id = unified_id + row.created_by = created_by + row.team_id = team_id + row.file_object = {"status": "completed", "input_file_id": "file-managed-1"} + row.file_purpose = "batch" + payload = decode(unified_id) + row.model_object_id = ( + f"passthrough:{payload.provider}:{payload.raw_provider_id}" + if payload is not None + else None + ) + + import datetime + + row.created_at = datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc) + return row + + +class TestListPassthroughIdsFromDb: + """Tests for list_passthrough_ids_from_db and is_passthrough_list_route.""" + + def test_is_passthrough_list_route_files(self): + assert is_passthrough_list_route("openai", "GET", "/openai/v1/files") is True + + def test_is_passthrough_list_route_batches(self): + assert ( + is_passthrough_list_route("azure", "GET", "/azure/openai/batches") is True + ) + + def test_is_passthrough_list_route_not_for_post(self): + assert is_passthrough_list_route("openai", "POST", "/openai/v1/files") is False + + def test_is_passthrough_list_route_not_for_single_resource(self): + # GET /v1/files/{file_id} is not a list route + assert ( + is_passthrough_list_route("openai", "GET", "/openai/v1/files/file-abc") + is False + ) + + def test_is_passthrough_list_route_azure_ai_prefix(self): + assert ( + is_passthrough_list_route("azure", "GET", "/azure_ai/openai/files") is True + ) + + def test_is_passthrough_list_route_azure_path_already_carrying_v1(self): + assert ( + is_passthrough_list_route("azure", "GET", "/azure/openai/v1/files") is True + ) + assert ( + is_passthrough_list_route("azure", "GET", "/azure/openai/v1/batches") + is True + ) + + def test_is_passthrough_list_route_not_for_azure_single_resource(self): + assert ( + is_passthrough_list_route("azure", "GET", "/azure/openai/files/file-abc") + is False + ) + + @pytest.mark.asyncio + async def test_list_files_returns_owned_rows(self): + managed_id = new_managed_id("openai", "file-abc") + fake_row = _fake_file_row(managed_id) + pc = _prisma_with_list(file_rows=[fake_row]) + + result = await list_passthrough_ids_from_db( + provider="openai", + route="/openai/v1/files", + user_api_key_dict=_user("user-1", "team-1"), + prisma_client=pc, + ) + + assert result is not None + assert result["object"] == "list" + assert len(result["data"]) == 1 + assert result["data"][0]["id"] == managed_id + assert result["data"][0]["object"] == "file" + assert result["first_id"] == managed_id + + @pytest.mark.asyncio + async def test_list_batches_returns_owned_rows(self): + managed_id = new_managed_id("openai", "batch_abc") + fake_row = _fake_batch_row(managed_id) + pc = _prisma_with_list(batch_rows=[fake_row]) + + result = await list_passthrough_ids_from_db( + provider="openai", + route="/openai/v1/batches", + user_api_key_dict=_user("user-1", "team-1"), + prisma_client=pc, + ) + + assert result is not None + assert result["object"] == "list" + assert len(result["data"]) == 1 + assert result["data"][0]["id"] == managed_id + assert result["data"][0]["object"] == "batch" + + @pytest.mark.asyncio + async def test_list_files_admin_gets_all_rows(self): + """Admin should receive all rows; the where filter passed to DB is {}.""" + rows = [ + _fake_file_row(new_managed_id("openai", "file-1")), + _fake_file_row(new_managed_id("openai", "file-2")), + ] + pc = _prisma_with_list(file_rows=rows) + + result = await list_passthrough_ids_from_db( + provider="openai", + route="/openai/v1/files", + user_api_key_dict=_admin_user(), + prisma_client=pc, + ) + + assert result is not None + assert len(result["data"]) == 2 + # Admin adds no owner scoping, but the provider scope is always pushed + # to the DB; the only where clause is the provider marker filter. + call_kwargs = pc.db.litellm_managedfiletable.find_many.call_args.kwargs + assert call_kwargs["where"] == { + "flat_model_file_ids": {"has": _passthrough_provider_marker("openai")} + } + + @pytest.mark.asyncio + async def test_list_files_user_scoped_where(self): + """Regular user should get a where clause scoped to their user_id / team_id.""" + pc = _prisma_with_list(file_rows=[]) + + await list_passthrough_ids_from_db( + provider="openai", + route="/openai/v1/files", + user_api_key_dict=_user("user-2", "team-2"), + prisma_client=pc, + ) + + call_kwargs = pc.db.litellm_managedfiletable.find_many.call_args.kwargs + where = call_kwargs["where"] + # The OR clause should scope to user-2 or team-2 + assert "OR" in where + entries = where["OR"] + assert {"created_by": "user-2"} in entries + assert {"team_id": "team-2"} in entries + + @pytest.mark.asyncio + async def test_list_has_more_flag(self): + """has_more is True when DB returns limit+1 rows.""" + rows = [ + _fake_file_row(new_managed_id("openai", f"file-{i}")) for i in range(21) + ] # limit=20, fetch 21 + pc = _prisma_with_list(file_rows=rows) + + result = await list_passthrough_ids_from_db( + provider="openai", + route="/openai/v1/files", + user_api_key_dict=_admin_user(), + prisma_client=pc, + query_params={"limit": "20"}, + ) + + assert result is not None + assert result["has_more"] is True + assert len(result["data"]) == 20 # extra row trimmed + + @pytest.mark.asyncio + async def test_list_returns_none_for_non_list_route(self): + pc = _prisma_with_list() + + result = await list_passthrough_ids_from_db( + provider="openai", + route="/openai/v1/files/file-abc", # single-resource, not a list + user_api_key_dict=_user(), + prisma_client=pc, + ) + + assert result is None + + @pytest.mark.asyncio + async def test_list_db_error_returns_empty_not_none(self): + """DB failure must return an empty list, not None (which would fall through + to the upstream provider and leak the provider-wide listing).""" + pc = _prisma_with_list() + pc.db.litellm_managedfiletable.find_many = AsyncMock( + side_effect=Exception("db down") + ) + + result = await list_passthrough_ids_from_db( + provider="openai", + route="/openai/v1/files", + user_api_key_dict=_admin_user(), + prisma_client=pc, + ) + + # Must not return None (which would fall through to upstream) + assert result is not None + assert result["data"] == [] + assert result["has_more"] is False + + @pytest.mark.asyncio + async def test_list_returns_empty_for_caller_without_identity(self): + """Caller with neither user_id nor team_id should get an empty list.""" + pc = _prisma_with_list( + file_rows=[_fake_file_row(new_managed_id("openai", "file-1"))] + ) + anon = UserAPIKeyAuth() # no user_id, no team_id, not admin + + result = await list_passthrough_ids_from_db( + provider="openai", + route="/openai/v1/files", + user_api_key_dict=anon, + prisma_client=pc, + ) + + assert result is not None + assert result["data"] == [] + + @pytest.mark.asyncio + async def test_list_files_pushes_provider_scope_to_db(self): + """File listing scopes by provider at the DB level via the provider + marker in flat_model_file_ids, so a single query serves the page and a + mixed-provider pool can never truncate or leak the other provider. + + A large azure-only pool must return an empty openai page with + has_more=False in exactly one DB round-trip. + """ + azure_rows = [ + _fake_file_row(new_managed_id("azure", f"file-{i}")) for i in range(50) + ] + pc = _prisma_with_list(file_rows=azure_rows) + + result = await list_passthrough_ids_from_db( + provider="openai", # asking for openai but DB only has azure rows + route="/openai/v1/files", + user_api_key_dict=_admin_user(), + prisma_client=pc, + query_params={"limit": "20"}, + ) + + assert result is not None + assert result["data"] == [] + assert result["has_more"] is False + where = pc.db.litellm_managedfiletable.find_many.call_args.kwargs["where"] + assert where["flat_model_file_ids"] == { + "has": _passthrough_provider_marker("openai") + } + assert pc.db.litellm_managedfiletable.find_many.await_count == 1 + + @pytest.mark.asyncio + async def test_list_ignores_cross_provider_cursor(self): + """An ``after`` cursor minted for a different provider must not shift the + created_at boundary: it would skip/repeat this provider's rows. The + cursor is ignored and the unscoped first page is served.""" + import datetime + + azure_row = _fake_file_row(new_managed_id("azure", "file-azure")) + pc = _prisma_with_list(file_rows=[azure_row]) + + cursor_row = MagicMock() + cursor_row.created_at = datetime.datetime( + 2025, 6, 1, tzinfo=datetime.timezone.utc + ) + pc.db.litellm_managedfiletable.find_first = AsyncMock(return_value=cursor_row) + + result = await list_passthrough_ids_from_db( + provider="azure", + route="/azure/openai/files", + user_api_key_dict=_admin_user(), + prisma_client=pc, + query_params={"after": new_managed_id("openai", "file-openai")}, + ) + + assert result is not None + where = pc.db.litellm_managedfiletable.find_many.call_args.kwargs["where"] + assert "created_at" not in where + assert "OR" not in where and "AND" not in where + + @pytest.mark.asyncio + async def test_list_applies_same_provider_cursor(self): + """An ``after`` cursor minted for the same provider advances pagination + past the cursor row using a compound (created_at, id) boundary so rows + sharing the cursor row's timestamp are not skipped.""" + import datetime + + azure_row = _fake_file_row(new_managed_id("azure", "file-azure")) + pc = _prisma_with_list(file_rows=[azure_row]) + + cursor_row = MagicMock() + cursor_row.created_at = datetime.datetime( + 2025, 6, 1, tzinfo=datetime.timezone.utc + ) + pc.db.litellm_managedfiletable.find_first = AsyncMock(return_value=cursor_row) + + cursor_id = new_managed_id("azure", "file-cursor") + result = await list_passthrough_ids_from_db( + provider="azure", + route="/azure/openai/files", + user_api_key_dict=_admin_user(), + prisma_client=pc, + query_params={"after": cursor_id}, + ) + + assert result is not None + where = pc.db.litellm_managedfiletable.find_many.call_args.kwargs["where"] + assert "created_at" not in where + assert where["OR"] == [ + {"created_at": {"lt": cursor_row.created_at}}, + { + "AND": [ + {"created_at": cursor_row.created_at}, + {"unified_file_id": {"lt": cursor_id}}, + ] + }, + ] + + @pytest.mark.asyncio + async def test_list_cursor_does_not_drop_created_at_ties(self): + """Regression: paginating a pool whose rows all share one created_at must + return every row exactly once. A timestamp-only ``lt`` cursor boundary + would skip every tied row after the first page; the compound + (created_at, id) boundary keeps the walk complete.""" + import datetime + + shared_ts = datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc) + rows = [_fake_file_row(new_managed_id("azure", f"file-{i}")) for i in range(5)] + for row in rows: + row.created_at = shared_ts + all_ids = {row.unified_file_id for row in rows} + + def _matches(row, where): + for key, cond in where.items(): + if key == "AND": + if not all(_matches(row, c) for c in cond): + return False + elif key == "OR": + if not any(_matches(row, c) for c in cond): + return False + elif key == "flat_model_file_ids": + marker = (cond or {}).get("has") + if marker not in (getattr(row, "flat_model_file_ids", None) or []): + return False + else: + actual = getattr(row, key, None) + if isinstance(cond, dict): + for op, val in cond.items(): + if op == "lt" and not (actual is not None and actual < val): + return False + if op == "gt" and not (actual is not None and actual > val): + return False + if op == "startswith" and not str(actual or "").startswith( + val + ): + return False + elif actual != cond: + return False + return True + + def _find_many(*_a, where=None, order=None, take=None, **_k): + matched = [r for r in rows if _matches(r, where or {})] + for spec in reversed(order or []): + ((field, direction),) = spec.items() + matched.sort( + key=lambda r: getattr(r, field), reverse=(direction == "desc") + ) + return matched if take is None else matched[:take] + + def _find_first(*_a, where=None, **_k): + return next((r for r in rows if _matches(r, where or {})), None) + + pc = _prisma_client() + pc.db.litellm_managedfiletable.find_many = AsyncMock(side_effect=_find_many) + pc.db.litellm_managedfiletable.find_first = AsyncMock(side_effect=_find_first) + + collected: list = [] + after = None + for _ in range(len(rows) + 2): + params = {"limit": "2"} + if after is not None: + params["after"] = after + result = await list_passthrough_ids_from_db( + provider="azure", + route="/azure/openai/files", + user_api_key_dict=_admin_user(), + prisma_client=pc, + query_params=params, + ) + assert result is not None + collected.extend(item["id"] for item in result["data"]) + if not result["has_more"]: + break + after = result["last_id"] + + assert sorted(collected) == sorted(all_ids) + assert len(collected) == len(set(collected)) + + @pytest.mark.asyncio + async def test_list_files_filters_by_provider(self): + openai_row = _fake_file_row(new_managed_id("openai", "file-openai")) + azure_row = _fake_file_row(new_managed_id("azure", "file-azure")) + pc = _prisma_with_list(file_rows=[azure_row, openai_row]) + + result = await list_passthrough_ids_from_db( + provider="openai", + route="/openai/v1/files", + user_api_key_dict=_admin_user(), + prisma_client=pc, + ) + + assert result is not None + assert len(result["data"]) == 1 + assert decode(result["data"][0]["id"]).provider == "openai" + + @pytest.mark.asyncio + async def test_list_batches_pushes_provider_scope_to_db(self): + """Batch listing scopes by provider at the DB level via the namespaced + model_object_id, so a single query serves the page instead of scanning.""" + batch_row = _fake_batch_row(new_managed_id("azure", "batch_abc")) + pc = _prisma_with_list(batch_rows=[batch_row]) + + result = await list_passthrough_ids_from_db( + provider="azure", + route="/azure/openai/batches", + user_api_key_dict=_admin_user(), + prisma_client=pc, + ) + + assert result is not None + assert len(result["data"]) == 1 + where = pc.db.litellm_managedobjecttable.find_many.call_args.kwargs["where"] + assert where["model_object_id"] == {"startswith": "passthrough:azure:"} + assert pc.db.litellm_managedobjecttable.find_many.await_count == 1