ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Litellm improve endpoint discovery (#18762)

Browse Source 
* docs: document all endpoints in .json and add consistency checks against docs + providers.json

* docs: add more tests + improve coverage
This commit is contained in:
Krish Dholakia 2026-01-07 17:35:01 +05:30 committed by GitHub
parent bdbbc9db62
commit 80ead21c3a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 1083 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.circleci/config.yml
View File

@ -1960,6 +1960,7 @@ jobs:
- run: ruff check ./litellm
# - run: python ./tests/documentation_tests/test_general_setting_keys.py
- run: python ./tests/code_coverage_tests/check_licenses.py
- run: python ./tests/code_coverage_tests/check_provider_folders_documented.py
- run: python ./tests/code_coverage_tests/router_code_coverage.py
- run: python ./tests/code_coverage_tests/test_chat_completion_imports.py
- run: python ./tests/code_coverage_tests/info_log_check.py

2
docs/my-website/docs/interactions.md
View File

@ -8,7 +8,7 @@ import TabItem from '@theme/TabItem';
| Logging | ✅ | Works across all integrations |
| Streaming | ✅ | |
| Loadbalancing | ✅ | Between supported models |
| Supported LLM providers | **All LiteLLM supported providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai` etc. |
| Supported LLM providers | **All LiteLLM supported CHAT COMPLETION providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai` etc. |
## **LiteLLM Python SDK Usage**

6
docs/my-website/docs/realtime.md
View File

@ -5,6 +5,12 @@ import TabItem from '@theme/TabItem';
Use this to loadbalance across Azure + OpenAI.
Supported Providers:
- OpenAI
- Azure
- Google AI Studio (Gemini)
- Vertex AI
## Proxy Usage
### Add model to config

53
docs/my-website/sidebars.js
View File

@ -420,14 +420,8 @@ const sidebars = {
],
},
"assistants",
{
type: "category",
label: "/audio",
items: [
"audio_transcription",
"text_to_speech",
]
},
"audio_transcription",
"text_to_speech",
{
type: "category",
label: "/batches",
@ -477,17 +471,13 @@ const sidebars = {
"apply_guardrail",
"bedrock_invoke",
"interactions",
{
type: "category",
label: "/images",
items: [
"image_edits",
"image_generation",
"image_variations",
]
},
"image_edits",
"image_generation",
"image_variations",
"videos",
"vector_store_files",
"vector_stores/create",
"vector_stores/search",
{
type: "category",
label: "/mcp - Model Context Protocol",
@ -531,24 +521,12 @@ const sidebars = {
"proxy/pass_through_guardrails"
]
},
{
type: "category",
label: "/rag",
items: [
"rag_ingest",
"rag_query",
]
},
"rag_ingest",
"rag_query",
"realtime",
"rerank",
{
type: "category",
label: "/responses",
items: [
"response_api",
"response_api_compact",
]
},
"response_api",
"response_api_compact",
{
type: "category",
label: "/search",
@ -566,14 +544,7 @@ const sidebars = {
]
},
"skills",
{
type: "category",
label: "/vector_stores",
items: [
"vector_stores/create",
"vector_stores/search",
]
},
],
},
{

457
provider_endpoints_support.json
View File

@ -20,16 +20,14 @@
"skills": "Supports /skills endpoint",
"interactions": "Supports /interactions endpoint (Google AI Interactions API)",
"a2a_(Agent Gateway)": "Supports /a2a/{agent}/message/send endpoint (A2A Protocol)",
"create_container": "Supports POST /containers endpoint",
"list_containers": "Supports GET /containers endpoint",
"retrieve_container": "Supports GET /containers/{id} endpoint",
"delete_container": "Supports DELETE /containers/{id} endpoint",
"create_container_file": "Supports POST /containers/{id}/files endpoint",
"list_container_files": "Supports GET /containers/{id}/files endpoint",
"retrieve_container_file": "Supports GET /containers/{id}/files/{file_id} endpoint",
"retrieve_container_file_content": "Supports GET /containers/{id}/files/{file_id}/content endpoint",
"delete_container_file": "Supports DELETE /containers/{id}/files/{file_id} endpoint",
"compact": "Supports /responses/compact endpoint"
"container": "Supports OpenAI's /containers endpoint",
"container_file": "Supports OpenAI's /containers/{id}/files endpoint",
"compact": "Supports /responses/compact endpoint",
"files": "Supports /files endpoint for file operations",
"image_edits": "Supports /images/edits endpoint for image editing",
"vector_stores_create": "Supports creating a new vector store via /vector_stores endpoint",
"vector_stores_search": "Supports searching a vector store via /vector_stores/{id}/search endpoint",
"video_generations": "Supports /videos/generations endpoint for video generation"
}
}
},
@ -122,7 +120,8 @@
"rerank": false,
"skills": true,
"a2a": true,
"interactions": true
"interactions": true,
"count_tokens": true
}
},
"anthropic_text": {
@ -211,7 +210,13 @@
"batches": false,
"rerank": true,
"a2a": true,
"interactions": true
"interactions": true,
"bedrock_invoke": true,
"bedrock_converse": true,
"vector_stores_search": true,
"count_tokens": true,
"rag_ingest": true,
"rag_query": true
}
},
"sagemaker": {
@ -263,7 +268,11 @@
"batches": true,
"rerank": false,
"a2a": true,
"interactions": true
"interactions": true,
"vector_stores_search": true,
"assistants": true,
"fine_tuning": true,
"text_completion": true
}
},
"azure_ai": {
@ -282,7 +291,9 @@
"rerank": false,
"ocr": true,
"a2a": true,
"interactions": true
"interactions": true,
"vector_stores_create": true,
"vector_stores_search": true
}
},
"azure_ai/doc-intelligence": {
@ -918,29 +929,19 @@
"embeddings": true,
"image_generations": true,
"audio_transcriptions": false,
"audio_speech": false,
"audio_speech": true,
"moderations": false,
"batches": false,
"rerank": false,
"ocr": true,
"a2a": true,
"interactions": true
}
},
"vertex_ai/chirp": {
"display_name": "Google - Vertex AI Chirp3 HD (`vertex_ai/chirp`)",
"url": "https://docs.litellm.ai/docs/providers/vertex_speech",
"endpoints": {
"chat_completions": false,
"messages": false,
"responses": false,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": true,
"moderations": false,
"batches": false,
"rerank": false
"interactions": true,
"vector_stores_search": true,
"count_tokens": true,
"fine_tuning": true,
"rag_ingest": true,
"rag_query": true,
"generateContent": true
}
},
"gemini": {
@ -958,7 +959,12 @@
"batches": false,
"rerank": false,
"interactions": true,
"a2a": true
"a2a": true,
"vector_stores_search": true,
"count_tokens": true,
"rag_ingest": true,
"realtime": true,
"generateContent": true
}
},
"gradient_ai": {
@ -1511,18 +1517,21 @@
"moderations": true,
"batches": true,
"rerank": false,
"create_container": true,
"list_containers": true,
"retrieve_container": true,
"delete_container": true,
"create_container_file": true,
"list_container_files": true,
"retrieve_container_file": true,
"retrieve_container_file_content": true,
"delete_container_file": true,
"container": true,
"compact": true,
"a2a": true,
"interactions": true
"interactions": true,
"vector_store_files": true,
"vector_stores_create": true,
"vector_stores_search": true,
"assistants": true,
"container_files": true,
"fine_tuning": true,
"image_variations": true,
"rag_ingest": true,
"rag_query": true,
"realtime": true,
"text_completion": true
}
},
"openai_like": {
@ -1538,7 +1547,8 @@
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false
"rerank": false,
"assistants": true
}
},
"openrouter": {
@ -1897,34 +1907,13 @@
"display_name": "Topaz (`topaz`)",
"url": "https://docs.litellm.ai/docs/providers/topaz",
"endpoints": {
"chat_completions": true,
"messages": true,
"responses": true,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"a2a": true,
"interactions": true
"image_variations": true
}
},
"tavily": {
"display_name": "Tavily (`tavily`)",
"url": "https://docs.litellm.ai/docs/search/tavily",
"endpoints": {
"chat_completions": false,
"messages": false,
"responses": false,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"search": true
}
},
@ -2137,7 +2126,7 @@
"moderations": false,
"batches": false,
"rerank": false,
"vector_stores": true,
"vector_stores_create": true,
"a2a": true,
"interactions": true
}
@ -2247,6 +2236,340 @@
"a2a": true,
"interactions": true
}
},
"gigachat": {
"display_name": "GigaChat (`gigachat`)",
"url": "https://docs.litellm.ai/docs/providers/gigachat",
"endpoints": {
"chat_completions": true,
"messages": true,
"responses": true,
"embeddings": true
}
},
"google_pse": {
"display_name": "Google PSE (`google_pse`)",
"url": "https://docs.litellm.ai/docs/search/google_pse",
"endpoints": {
"search": true
}
},
"milvus": {
"display_name": "Milvus (`milvus`)",
"url": "https://docs.litellm.ai/docs/providers/milvus_vector_stores",
"endpoints": {
"vector_stores_search": true
}
},
"minimax": {
"display_name": "Minimax (`minimax`)",
"url": "https://docs.litellm.ai/docs/providers/minimax",
"endpoints": {
"chat_completions": true,
"messages": true,
"responses": true
}
},
"pg_vector": {
"display_name": "PG Vector (`pg_vector`)",
"url": "https://docs.litellm.ai/docs/providers/pg_vector",
"endpoints": {
"vector_stores_search": true
}
},
"helicone": {
"display_name": "Helicone (`helicone`)",
"url": "https://docs.litellm.ai/docs/providers/helicone",
"endpoints": {
"chat_completions": true,
"messages": true,
"responses": true
}
},
"llamagate": {
"display_name": "LlamaGate (`llamagate`)",
"url": "https://docs.litellm.ai/docs/providers/llamagate",
"endpoints": {
"chat_completions": true,
"messages": true,
"responses": true
}
},
"xiaomi_mimo": {
"display_name": "Xiaomi Mimo (`xiaomi_mimo`)",
"url": "https://docs.litellm.ai/docs/providers/xiaomi_mimo",
"endpoints": {
"chat_completions": true,
"messages": true,
"responses": true
}
}
},
"endpoints": {
"a2a": {
"docs_label": "a2a",
"display_name": "A2A (Agent-to-Agent) protocol for agent communication",
"leftnav_label": "/a2a",
"provider_json_field": "a2a",
"url": "https://docs.litellm.ai/docs/a2a",
"bridges_to_chat_completion": true
},
"messages": {
"docs_label": "anthropic_unified",
"display_name": "Anthropic /v1/messages API",
"leftnav_label": "/messages",
"provider_json_field": "messages",
"url": "https://docs.litellm.ai/docs/anthropic_unified",
"bridges_to_chat_completion": true
},
"anthropic_count_tokens": {
"docs_label": "anthropic_count_tokens",
"display_name": "Anthropic /v1/messages/count_tokens API",
"leftnav_label": "/count_tokens",
"provider_json_field": "count_tokens",
"url": "https://docs.litellm.ai/docs/anthropic_count_tokens"
},
"apply_guardrail": {
"docs_label": "apply_guardrail",
"display_name": "Unified Apply Guardrail API",
"leftnav_label": "/guardrails/apply_guardrail",
"provider_json_field": "apply_guardrail",
"url": "https://docs.litellm.ai/docs/apply_guardrail"
},
"assistants": {
"docs_label": "assistants",
"display_name": "OpenAI Assistants API",
"leftnav_label": "/assistants",
"provider_json_field": "assistants",
"url": "https://docs.litellm.ai/docs/assistants"
},
"audio_transcription": {
"docs_label": "audio_transcription",
"display_name": "Audio Transcription API",
"leftnav_label": "/audio/transcriptions",
"provider_json_field": "audio_transcriptions",
"url": "https://docs.litellm.ai/docs/audio_transcription"
},
"batches": {
"docs_label": "batches",
"display_name": "Batches API",
"leftnav_label": "/batches",
"provider_json_field": "batches",
"url": "https://docs.litellm.ai/docs/batches"
},
"bedrock_invoke": {
"docs_label": "bedrock_invoke",
"display_name": "Bedrock Invoke API",
"leftnav_label": "/invoke",
"provider_json_field": "bedrock_invoke",
"url": "https://docs.litellm.ai/docs/bedrock_invoke"
},
"bedrock_converse": {
"docs_label": "bedrock_converse",
"display_name": "Bedrock Converse API",
"leftnav_label": "/converse",
"provider_json_field": "bedrock_converse",
"url": "https://docs.litellm.ai/docs/bedrock_converse"
},
"chat_completions": {
"docs_label": "chat_completions",
"display_name": "Chat Completions API",
"leftnav_label": "/chat/completions",
"provider_json_field": "chat_completions",
"url": "https://docs.litellm.ai/docs/chat_completions"
},
"container_files": {
"docs_label": "container_files",
"display_name": "OpenAI Container Files API",
"leftnav_label": "/create/container/files",
"provider_json_field": "container_files",
"url": "https://docs.litellm.ai/docs/container_files"
},
"container": {
"docs_label": "containers",
"display_name": "OpenAI Containers API",
"leftnav_label": "/container",
"provider_json_field": "container",
"url": "https://docs.litellm.ai/docs/containers"
},
"embeddings": {
"docs_label": "embedding/supported_embedding",
"display_name": "Embedding API (OpenAI Format)",
"leftnav_label": "/embeddings",
"provider_json_field": "embeddings",
"url": "https://docs.litellm.ai/docs/embedding/supported_embedding"
},
"files": {
"docs_label": "files",
"display_name": "OpenAI Files API",
"leftnav_label": "/files",
"provider_json_field": "files",
"url": "https://docs.litellm.ai/docs/proxy/litellm_managed_files"
},
"fine_tuning": {
"docs_label": "fine_tuning",
"display_name": "OpenAI Fine-Tuning API",
"leftnav_label": "/fine_tuning",
"provider_json_field": "fine_tuning",
"url": "https://docs.litellm.ai/docs/proxy/managed_finetuning"
},
"generateContent": {
"docs_label": "generateContent",
"display_name": "Google's GenerateContent API",
"leftnav_label": "/generateContent",
"provider_json_field": "generateContent",
"url": "https://docs.litellm.ai/docs/generateContent",
"bridges_to_chat_completion": true
},
"image_edits": {
"docs_label": "image_edits",
"display_name": "OpenAI Images Edits API",
"leftnav_label": "/images/edits",
"provider_json_field": "image_edits",
"url": "https://docs.litellm.ai/docs/image_edits"
},
"image_generations": {
"docs_label": "image_generation",
"display_name": "OpenAI Images Generations API",
"leftnav_label": "/images/generations",
"provider_json_field": "image_generations",
"url": "https://docs.litellm.ai/docs/image_generation"
},
"image_variations": {
"docs_label": "image_variations",
"display_name": "OpenAI Images Variations API",
"leftnav_label": "/images/variations",
"provider_json_field": "image_variations",
"url": "https://docs.litellm.ai/docs/image_variations"
},
"interactions": {
"docs_label": "interactions",
"display_name": "Google Interactions API",
"leftnav_label": "/interactions",
"provider_json_field": "interactions",
"url": "https://docs.litellm.ai/docs/interactions",
"bridges_to_chat_completion": true
},
"mcp": {
"docs_label": "mcp",
"display_name": "Model Context Protocol (MCP)",
"leftnav_label": "/mcp",
"provider_json_field": "mcp",
"url": "https://docs.litellm.ai/docs/mcp"
},
"moderation": {
"docs_label": "moderation",
"display_name": "OpenAI Moderation API",
"leftnav_label": "/moderations",
"provider_json_field": "moderations",
"url": "https://docs.litellm.ai/docs/moderation"
},
"ocr": {
"docs_label": "ocr",
"display_name": "OCR API (Mistral Format)",
"leftnav_label": "/ocr",
"provider_json_field": "ocr",
"url": "https://docs.litellm.ai/docs/ocr"
},
"rag_ingest": {
"docs_label": "rag_ingest",
"display_name": "RAG Ingest API",
"leftnav_label": "/rag/ingest",
"provider_json_field": "rag_ingest",
"url": "https://docs.litellm.ai/docs/rag_ingest"
},
"rag_query": {
"docs_label": "rag_query",
"display_name": "RAG Query API",
"leftnav_label": "/rag/query",
"provider_json_field": "rag_query",
"url": "https://docs.litellm.ai/docs/rag_query"
},
"realtime": {
"docs_label": "realtime",
"display_name": "OpenAI Realtime API",
"leftnav_label": "/realtime",
"provider_json_field": "realtime",
"url": "https://docs.litellm.ai/docs/realtime"
},
"rerank": {
"docs_label": "rerank",
"display_name": "Rerank API (Cohere Format)",
"leftnav_label": "/rerank",
"provider_json_field": "rerank",
"url": "https://docs.litellm.ai/docs/rerank"
},
"responses": {
"docs_label": "response_api",
"display_name": "Responses API (OpenAI Format)",
"leftnav_label": "/responses",
"provider_json_field": "responses",
"url": "https://docs.litellm.ai/docs/response_api",
"bridges_to_chat_completion": true
},
"response_api_compact": {
"docs_label": "response_api_compact",
"display_name": "Responses API (OpenAI Format)",
"leftnav_label": "/responses",
"provider_json_field": "compact",
"url": "https://docs.litellm.ai/docs/response_api"
},
"search": {
"docs_label": "search",
"display_name": "Search API",
"leftnav_label": "/search",
"provider_json_field": "search",
"url": "https://docs.litellm.ai/docs/search"
},
"skills": {
"docs_label": "skills",
"display_name": "Anthropic Skills API",
"leftnav_label": "/skills",
"provider_json_field": "skills",
"url": "https://docs.litellm.ai/docs/skills"
},
"text_completion": {
"docs_label": "text_completion",
"display_name": "Completions API (OpenAI Format)",
"leftnav_label": "/completions",
"provider_json_field": "text_completion",
"url": "https://docs.litellm.ai/docs/text_completion",
"bridges_to_chat_completion": true
},
"text_to_speech": {
"docs_label": "text_to_speech",
"display_name": "Text-to-Speech API (OpenAI Format)",
"leftnav_label": "/audio/speech",
"provider_json_field": "audio_speech",
"url": "https://docs.litellm.ai/docs/text_to_speech"
},
"vector_store_files": {
"docs_label": "vector_store_files",
"display_name": "OpenAI Vector Store Files API",
"leftnav_label": "/vector_stores/files",
"provider_json_field": "vector_store_files",
"url": "https://docs.litellm.ai/docs/vector_store_files"
},
"vector_stores_create": {
"docs_label": "vector_stores_create",
"display_name": "OpenAI Vector Stores Create API",
"leftnav_label": "/vector_stores/create",
"provider_json_field": "vector_stores_create",
"url": "https://docs.litellm.ai/docs/vector_stores/create"
},
"vector_stores_search": {
"docs_label": "vector_stores_search",
"display_name": "OpenAI Vector Stores Search API",
"leftnav_label": "/vector_stores/search",
"provider_json_field": "vector_stores_search",
"url": "https://docs.litellm.ai/docs/vector_stores/search"
},
"videos": {
"docs_label": "videos",
"display_name": "OpenAI Video Generation API",
"leftnav_label": "/videos",
"provider_json_field": "video_generations",
"url": "https://docs.litellm.ai/docs/videos"
}
}
}

379
tests/code_coverage_tests/check_endpoint_coverage.py Normal file
View File

@ -0,0 +1,379 @@
"""
Code coverage test to ensure all endpoints documented in sidebars.js are defined in provider_endpoints_support.json.
This script:
1. Extracts all endpoint entries from the "Supported Endpoints" section of sidebars.js
2. Validates that each endpoint has a corresponding entry in the "endpoints" object of provider_endpoints_support.json
3. Checks that the "docs_label" field is present in each endpoint definition
"""
import json
import re
import sys
from pathlib import Path
from typing import Dict, List, Set, Tuple
class MissingEndpointDefinitionError(Exception):
"""Raised when endpoints are documented in sidebars.js but missing from provider_endpoints_support.json."""
pass
def get_repo_root() -> Path:
"""Get the repository root directory."""
# Check if litellm directory exists in current working directory
cwd = Path.cwd()
if (cwd / "litellm").exists() and (cwd / "litellm").is_dir():
# We're already at the repo root
return cwd
# Otherwise, navigate up from script location
current = Path(__file__).resolve()
# Navigate up from tests/code_coverage_tests/
return current.parent.parent.parent
def extract_endpoints_from_sidebars() -> Dict[str, str]:
"""
Extract endpoint entries from sidebars.js.
Returns a dict mapping endpoint_key -> label
Only extracts top-level endpoint entries from the "Supported Endpoints" section.
"""
repo_root = get_repo_root()
sidebars_path = repo_root / "docs" / "my-website" / "sidebars.js"
if not sidebars_path.exists():
print(f"❌ ERROR: Could not find sidebars.js at {sidebars_path}")
sys.exit(1)
with open(sidebars_path, "r") as f:
content = f.read()
# Find the Supported Endpoints section
supported_start = content.find('label: "Supported Endpoints"')
if supported_start == -1:
print("⚠️ WARNING: Could not find 'Supported Endpoints' section")
return {}
# Find the items array within this section
items_start = content.find("items: [", supported_start)
if items_start == -1:
print("⚠️ WARNING: Could not find items array in Supported Endpoints")
return {}
# Find the end of this items array
# Look for the closing ], at the same indentation level
items_end = content.find("\n ],\n },\n {", items_start)
if items_end == -1:
items_end = content.find("\n ],\n }", items_start)
section = content[items_start:items_end]
endpoints = {}
# Pattern 1: Categories with labels at the top level (8 spaces indent)
# Example: " {type: "category", label: "/a2a - A2A Agent Gateway""
category_pattern = (
r'^\s{8}\{\s*\n\s{10}type:\s*"category",\s*\n\s{10}label:\s*"([^"]+)"'
)
for match in re.finditer(category_pattern, section, re.MULTILINE):
label = match.group(1)
# Skip utility categories
if "Pass-through" in label or label == "Vertex AI":
continue
endpoint_key = label.split(" - ")[0].strip("/").replace("/", "_")
endpoints[endpoint_key] = label
# Pattern 2: Standalone doc strings at top level (8 spaces indent)
# Example: " "assistants","
standalone_pattern = r'^\s{8}"([a-zA-Z_][a-zA-Z0-9_]*)",?\s*$'
for match in re.finditer(standalone_pattern, section, re.MULTILINE):
doc_id = match.group(1)
endpoints[doc_id] = doc_id
return endpoints
def load_provider_endpoints_file() -> Dict:
"""Load the provider_endpoints_support.json file."""
repo_root = get_repo_root()
file_path = repo_root / "provider_endpoints_support.json"
if not file_path.exists():
print(
f"❌ ERROR: Could not find provider_endpoints_support.json at {file_path}"
)
sys.exit(1)
with open(file_path, "r") as f:
return json.load(f)
def get_defined_endpoints(data: Dict) -> Dict[str, Dict]:
"""Get all endpoint definitions from provider_endpoints_support.json."""
return data.get("endpoints", {})
def normalize_endpoint_key(key: str) -> Set[str]:
"""
Generate variations of an endpoint key for matching.
Examples:
- "a2a" -> {"a2a"}
- "chat_completions" -> {"chat_completions", "chatcompletions"}
- "vector_stores" -> {"vector_stores", "vectorstores"}
"""
variations = {key, key.replace("_", "")}
return variations
def check_provider_endpoint_keys(data: Dict) -> List[str]:
"""
Check that all endpoint keys used in providers are defined in the root endpoints section.
Returns a list of missing endpoint keys.
"""
# Collect all unique endpoint keys used across all providers
provider_endpoint_keys = set()
providers = data.get("providers", {})
for provider_name, provider_data in providers.items():
if "endpoints" in provider_data and isinstance(
provider_data["endpoints"], dict
):
provider_endpoint_keys.update(provider_data["endpoints"].keys())
# Get all endpoint definitions
defined_endpoints = data.get("endpoints", {})
# Collect all provider_json_field values from endpoint definitions
provider_json_fields = set()
for endpoint_key, endpoint_data in defined_endpoints.items():
if isinstance(endpoint_data, dict) and "provider_json_field" in endpoint_data:
provider_json_fields.add(endpoint_data["provider_json_field"])
# Find missing endpoint keys
missing_keys = []
for key in sorted(provider_endpoint_keys):
if key not in provider_json_fields:
missing_keys.append(key)
return missing_keys
def check_unused_endpoints(data: Dict) -> List[Tuple[str, str]]:
"""
Check that all defined endpoints are used by at least one provider.
Returns a list of tuples (endpoint_key, provider_json_field) for unused endpoints.
"""
# Special endpoints that don't need to be used by specific providers
# These are utility/framework endpoints available across the platform
SPECIAL_ENDPOINTS = {
"apply_guardrail", # Guardrail application - works across providers
"mcp", # Model Context Protocol - works across providers
}
# Get all endpoint definitions
defined_endpoints = data.get("endpoints", {})
providers = data.get("providers", {})
# Collect all endpoint keys used by providers
used_keys = set()
for provider_data in providers.values():
if "endpoints" in provider_data and isinstance(
provider_data["endpoints"], dict
):
used_keys.update(provider_data["endpoints"].keys())
# Find unused endpoints (excluding special ones)
unused = []
for endpoint_key, endpoint_data in defined_endpoints.items():
# Skip special endpoints
if endpoint_key in SPECIAL_ENDPOINTS:
continue
if isinstance(endpoint_data, dict) and "provider_json_field" in endpoint_data:
provider_json_field = endpoint_data["provider_json_field"]
# Check if this provider_json_field is used by any provider
if provider_json_field not in used_keys:
unused.append((endpoint_key, provider_json_field))
return sorted(unused)
def main():
"""Main function to validate endpoint coverage."""
print(
"🔍 Checking endpoint coverage between sidebars.js and provider_endpoints_support.json..."
)
has_errors = False
# Load provider_endpoints_support.json
data = load_provider_endpoints_file()
defined_endpoints = get_defined_endpoints(data)
# Test 1: Check that endpoints from sidebars.js have docs_label entries
print("\n📖 Test 1: Checking endpoints from sidebars.js...")
sidebar_endpoints = extract_endpoints_from_sidebars()
print(f"✓ Found {len(sidebar_endpoints)} endpoints in sidebars.js")
print(
f"✓ Found {len(defined_endpoints)} endpoint definitions in provider_endpoints_support.json"
)
# Check for missing endpoints
missing_endpoints = []
# Collect all docs_label values from defined endpoints
defined_docs_labels = set()
for endpoint_data in defined_endpoints.values():
if isinstance(endpoint_data, dict) and "docs_label" in endpoint_data:
defined_docs_labels.add(endpoint_data["docs_label"])
for sidebar_key, sidebar_label in sorted(sidebar_endpoints.items()):
# Generate variations for matching against docs_label
variations = normalize_endpoint_key(sidebar_key)
# Check if any variation exists in docs_label values
if not any(var in defined_docs_labels for var in variations):
missing_endpoints.append((sidebar_key, sidebar_label))
# Report missing endpoints from sidebars
if missing_endpoints:
has_errors = True
error_msg = "\n❌ ERROR: The following endpoints are in sidebars.js but missing from provider_endpoints_support.json:\n"
error_msg += "=" * 70 + "\n"
for key, label in missing_endpoints:
error_msg += f" - {key}\n"
error_msg += f' Label in sidebars.js: "{label}"\n'
error_msg += "\n" + "=" * 70 + "\n"
error_msg += f"\n💡 To fix: Add these {len(missing_endpoints)} endpoint(s) to the 'endpoints' object\n"
error_msg += " in provider_endpoints_support.json\n"
error_msg += "\nExample format:\n"
error_msg += ' "endpoints": {\n'
for key, label in missing_endpoints[:5]:
error_msg += f' "{key}": {{\n'
error_msg += f' "docs_label": "{label}",\n'
error_msg += f' "provider_json_field": "{key}",\n'
error_msg += f' "description": "Description of the {label} endpoint"\n'
error_msg += " },\n"
if len(missing_endpoints) > 5:
error_msg += " ...\n"
error_msg += " }\n"
print(error_msg)
else:
print(
f"✅ All {len(sidebar_endpoints)} endpoints from sidebars.js are defined!"
)
# Test 2: Check that all provider endpoint keys have provider_json_field entries
print("\n📋 Test 2: Checking provider endpoint keys...")
missing_provider_keys = check_provider_endpoint_keys(data)
if missing_provider_keys:
has_errors = True
error_msg = "\n❌ ERROR: The following endpoint keys are used in providers but missing provider_json_field definitions:\n"
error_msg += "=" * 70 + "\n"
for key in missing_provider_keys:
# Find which providers use this key
using_providers = []
for provider_name, provider_data in data.get("providers", {}).items():
if key in provider_data.get("endpoints", {}):
using_providers.append(provider_name)
error_msg += f" - {key}\n"
error_msg += f" Used by {len(using_providers)} provider(s): {', '.join(using_providers[:3])}"
if len(using_providers) > 3:
error_msg += f" and {len(using_providers) - 3} more"
error_msg += "\n"
error_msg += "\n" + "=" * 70 + "\n"
error_msg += f"\n💡 To fix: Add these {len(missing_provider_keys)} endpoint(s) to the 'endpoints' object\n"
error_msg += " in provider_endpoints_support.json with 'provider_json_field' matching the key\n"
error_msg += "\nExample format:\n"
error_msg += ' "endpoints": {\n'
for key in missing_provider_keys[:3]:
error_msg += f' "{key}": {{\n'
error_msg += f' "docs_label": "{key}",\n'
error_msg += f' "provider_json_field": "{key}",\n'
error_msg += f' "description": "Description of the {key} endpoint"\n'
error_msg += " },\n"
if len(missing_provider_keys) > 3:
error_msg += " ...\n"
error_msg += " }\n"
print(error_msg)
else:
print("✅ All provider endpoint keys have provider_json_field definitions!")
# Test 3: Check that all defined endpoints are used by at least one provider
print("\n🔍 Test 3: Checking for unused endpoint definitions...")
unused_endpoints = check_unused_endpoints(data)
if unused_endpoints:
has_errors = True
error_msg = "\n⚠️ WARNING: The following endpoint definitions are not used by any provider:\n"
error_msg += "=" * 70 + "\n"
for endpoint_key, provider_json_field in unused_endpoints:
endpoint_data = defined_endpoints.get(endpoint_key, {})
docs_label = endpoint_data.get("docs_label", "N/A")
error_msg += f" - {endpoint_key}\n"
error_msg += f" provider_json_field: '{provider_json_field}'\n"
error_msg += f" docs_label: '{docs_label}'\n"
error_msg += "\n" + "=" * 70 + "\n"
error_msg += f"\n💡 These {len(unused_endpoints)} endpoint(s) are defined but not used by any provider.\n"
error_msg += " Either:\n"
error_msg += (
" 1. Add the endpoint to relevant providers' 'endpoints' objects, OR\n"
)
error_msg += " 2. Remove the endpoint definition if it's no longer needed\n"
print(error_msg)
else:
print("✅ All endpoint definitions are used by at least one provider!")
# Raise error if any tests failed
if has_errors:
error_summary = []
if missing_endpoints:
error_summary.append(f"{len(missing_endpoints)} endpoints from sidebars.js")
if missing_provider_keys:
error_summary.append(f"{len(missing_provider_keys)} provider endpoint keys")
if unused_endpoints:
error_summary.append(f"{len(unused_endpoints)} unused endpoint definitions")
raise MissingEndpointDefinitionError(
f"Endpoint validation failed: Missing definitions for {' and '.join(error_summary)}"
)
print("\n🎉 All endpoint coverage validations passed!")
return 0
if __name__ == "__main__":
try:
sys.exit(main())
except MissingEndpointDefinitionError as e:
print(f"\n🚨 CRITICAL ERROR: {e}\n")
sys.exit(1)
except Exception as e:
print(f"\n🚨 UNEXPECTED ERROR: {e}\n")
import traceback
traceback.print_exc()
sys.exit(1)

294
tests/code_coverage_tests/check_provider_folders_documented.py Normal file
View File

@ -0,0 +1,294 @@
"""
Code coverage test to ensure all provider folders are documented.
This script validates that:
1. Every provider folder in litellm/llms/ has a corresponding entry in provider_endpoints_support.json
2. Every provider in litellm/llms/openai_like/providers.json is documented in provider_endpoints_support.json
"""
import json
import os
import sys
from pathlib import Path
from typing import Dict, List, Set, Tuple
class UndocumentedProviderError(Exception):
"""Raised when providers are found without documentation."""
pass
# Special folders that should be excluded from validation
EXCLUDED_FOLDERS = {
"__pycache__",
"base_llm",
"deprecated_providers",
"custom_httpx",
"pass_through",
"openai_like", # This is a generic handler, not a specific provider
"aiohttp_openai", # Internal implementation detail for async HTTP
}
def get_repo_root() -> Path:
"""Get the repository root directory."""
# Check if litellm directory exists in current working directory
cwd = Path.cwd()
if (cwd / "litellm").exists() and (cwd / "litellm").is_dir():
# We're already at the repo root
return cwd
# Otherwise, navigate up from script location
current = Path(__file__).resolve()
# Navigate up from tests/code_coverage_tests/
return current.parent.parent.parent
def get_llm_provider_folders() -> Set[str]:
"""Get all provider folder names from litellm/llms directory."""
repo_root = get_repo_root()
llms_dir = repo_root / "litellm" / "llms"
if not llms_dir.exists():
print(f"❌ ERROR: Could not find llms directory at {llms_dir}")
sys.exit(1)
folders = set()
for item in llms_dir.iterdir():
if item.is_dir() and item.name not in EXCLUDED_FOLDERS:
folders.add(item.name)
return folders
def load_provider_endpoints_file() -> Dict:
"""Load the provider_endpoints_support.json file."""
repo_root = get_repo_root()
file_path = repo_root / "provider_endpoints_support.json"
if not file_path.exists():
print(
f"❌ ERROR: Could not find provider_endpoints_support.json at {file_path}"
)
sys.exit(1)
with open(file_path, "r") as f:
return json.load(f)
def get_openai_like_providers() -> Set[str]:
"""Get all provider names from litellm/llms/openai_like/providers.json."""
repo_root = get_repo_root()
providers_file = repo_root / "litellm" / "llms" / "openai_like" / "providers.json"
if not providers_file.exists():
print(
f"⚠️ WARNING: Could not find openai_like/providers.json at {providers_file}"
)
return set()
with open(providers_file, "r") as f:
data = json.load(f)
# Return all provider keys from the JSON
return set(data.keys())
def get_documented_providers(data: Dict) -> Set[str]:
"""Get all provider slugs documented in provider_endpoints_support.json."""
providers = data.get("providers", {})
# Get all provider keys, including those with slashes
documented = set()
for provider_key in providers.keys():
# For providers like "azure_ai/doc-intelligence", extract base name
base_name = provider_key.split("/")[0]
documented.add(base_name)
# Also add the full key in case folder name matches exactly
documented.add(provider_key)
return documented
def normalize_provider_name(folder_name: str) -> Set[str]:
"""
Generate possible provider names that might match a folder.
Some folders might have variations in the JSON:
- github_copilot folder -> github_copilot provider
- azure folder -> azure, azure_text, azure_ai providers
"""
variations = {folder_name}
# Add common variations
if "_" in folder_name:
# Try without underscores (though less common)
variations.add(folder_name.replace("_", ""))
return variations
def main():
"""Main function to validate provider documentation."""
print("🔍 Checking that all providers are documented...")
has_errors = False
# Check 1: Provider folders in litellm/llms
print("\n📁 Checking provider folders in litellm/llms/...")
provider_folders = get_llm_provider_folders()
print(f"✓ Found {len(provider_folders)} provider folders")
# Check 2: OpenAI-like providers
print("\n📋 Checking openai_like providers...")
openai_like_providers = get_openai_like_providers()
print(f"✓ Found {len(openai_like_providers)} openai_like providers")
# Load the JSON file
data = load_provider_endpoints_file()
documented_providers = get_documented_providers(data)
print(
f"\n✓ Found {len(data.get('providers', {}))} provider entries in provider_endpoints_support.json"
)
# Check for undocumented folders
undocumented_folders = []
for folder in sorted(provider_folders):
# Check if folder name or any variation is documented
variations = normalize_provider_name(folder)
if not any(var in documented_providers for var in variations):
undocumented_folders.append(folder)
# Check for undocumented openai_like providers
undocumented_openai_like = []
for provider in sorted(openai_like_providers):
# Generate multiple possible variations of the provider name
variations = {
provider, # Original name (e.g., "nano-gpt")
provider.replace(
"-", "_"
), # Replace hyphens with underscores (e.g., "nano_gpt")
provider.replace("-", ""), # Remove hyphens (e.g., "nanogpt")
provider.replace("_", ""), # Remove underscores
}
# Special case mappings for known variations
special_mappings = {
"veniceai": "venice",
"nano-gpt": "nanogpt",
}
if provider in special_mappings:
variations.add(special_mappings[provider])
# Check if any variation is documented
if not any(var in documented_providers for var in variations):
undocumented_openai_like.append(provider)
# Collect all error messages
error_messages: List[str] = []
# Report errors for undocumented folders
if undocumented_folders:
has_errors = True
error_msg = "\n❌ ERROR: The following provider folders are not documented:\n"
error_msg += "=" * 70 + "\n"
for folder in undocumented_folders:
error_msg += f" - litellm/llms/{folder}/\n"
error_msg += "\n" + "=" * 70 + "\n"
error_msg += f"\n💡 To fix: Add entries for these {len(undocumented_folders)} provider(s)\n"
error_msg += (
" in the 'providers' section of provider_endpoints_support.json\n"
)
error_msg += "\nExample format:\n"
error_msg += ' "providers": {\n'
for folder in undocumented_folders[:3]:
error_msg += f' "{folder}": {{\n'
error_msg += f' "display_name": "{folder.replace("_", " ").title()} (`{folder}`)",\n'
error_msg += (
f' "url": "https://docs.litellm.ai/docs/providers/{folder}",\n'
)
error_msg += ' "endpoints": {\n'
error_msg += ' "chat_completions": true,\n'
error_msg += ' "messages": true,\n'
error_msg += ' "responses": true,\n'
error_msg += ' "embeddings": false,\n'
error_msg += " ...\n"
error_msg += " }\n"
error_msg += " },\n"
if len(undocumented_folders) > 3:
error_msg += " ...\n"
error_msg += " }\n"
print(error_msg)
error_messages.append(
f"Found {len(undocumented_folders)} undocumented provider folders: {', '.join(undocumented_folders)}"
)
# Report errors for undocumented openai_like providers
if undocumented_openai_like:
has_errors = True
error_msg = (
"\n❌ ERROR: The following openai_like providers are not documented:\n"
)
error_msg += "=" * 70 + "\n"
for provider in undocumented_openai_like:
error_msg += f" - {provider}\n"
error_msg += "\n" + "=" * 70 + "\n"
error_msg += f"\n💡 To fix: Add entries for these {len(undocumented_openai_like)} provider(s)\n"
error_msg += (
" in the 'providers' section of provider_endpoints_support.json\n"
)
error_msg += "\nExample format:\n"
error_msg += ' "providers": {\n'
for provider in undocumented_openai_like[:3]:
normalized = provider.replace("-", "_")
error_msg += f' "{normalized}": {{\n'
error_msg += f' "display_name": "{provider.replace("-", " ").replace("_", " ").title()} (`{normalized}`)",\n'
error_msg += (
f' "url": "https://docs.litellm.ai/docs/providers/{normalized}",\n'
)
error_msg += ' "endpoints": {\n'
error_msg += ' "chat_completions": true,\n'
error_msg += ' "messages": true,\n'
error_msg += ' "responses": true,\n'
error_msg += ' "embeddings": false,\n'
error_msg += " ...\n"
error_msg += " }\n"
error_msg += " },\n"
if len(undocumented_openai_like) > 3:
error_msg += " ...\n"
error_msg += " }\n"
print(error_msg)
error_messages.append(
f"Found {len(undocumented_openai_like)} undocumented openai_like providers: {', '.join(undocumented_openai_like)}"
)
# Raise exception if there are any errors
if has_errors:
error_summary = " AND ".join(error_messages)
raise UndocumentedProviderError(
f"Provider documentation validation failed: {error_summary}"
)
print(f"\n✅ All {len(provider_folders)} provider folders are documented!")
print(f"✅ All {len(openai_like_providers)} openai_like providers are documented!")
print("\n🎉 All provider documentation checks passed!")
return 0
if __name__ == "__main__":
try:
sys.exit(main())
except UndocumentedProviderError as e:
print(f"\n🚨 CRITICAL ERROR: {e}\n")
sys.exit(1)
except Exception as e:
print(f"\n🚨 UNEXPECTED ERROR: {e}\n")
import traceback
traceback.print_exc()
sys.exit(1)