fix(router): enable order fallback for wildcard model groups
Use wildcard-aware deployment lookup when building order-based fallback levels so requests like openai/gpt-4.1-mini can advance from order=1 to order=2, and add a regression test for wildcard routing. Made-with: Cursor
This commit is contained in:
parent
72a461ba4a
commit
9ed90d53cd
@ -5358,9 +5358,11 @@ class Router:
|
||||
_request_team_id: Optional[str] = (
|
||||
kwargs.get("metadata", {}) or {}
|
||||
).get("user_api_key_team_id")
|
||||
all_deployments = self._get_all_deployments(
|
||||
# Use wildcard-aware lookup so order-based fallback also works for model
|
||||
# groups resolved via pattern routing (e.g. `openai/*` -> `openai/gpt-4.1-mini`).
|
||||
all_deployments = self.get_model_list(
|
||||
model_name=original_model_group, team_id=_request_team_id
|
||||
)
|
||||
) or []
|
||||
_order_set: set = {
|
||||
litellm.utils._get_deployment_order(d)
|
||||
for d in all_deployments
|
||||
|
||||
@ -329,3 +329,39 @@ async def test_router_order_fallback_with_non_standard_fallbacks():
|
||||
fallbacks=["fallback-model"], # non-standard format, passed per-request
|
||||
)
|
||||
assert response._hidden_params["model_id"] == "fallback"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_order_fallback_with_wildcard_model_group():
|
||||
"""Wildcard model groups should also advance across order levels."""
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": "bad",
|
||||
"mock_response": Exception("fail order 1"),
|
||||
"order": 1,
|
||||
},
|
||||
"model_info": {"id": "1"},
|
||||
},
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": "good",
|
||||
"mock_response": "success from wildcard order 2",
|
||||
"order": 2,
|
||||
},
|
||||
"model_info": {"id": "2"},
|
||||
},
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
response = await router.acompletion(
|
||||
model="openai/gpt-4.1-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
assert response._hidden_params["model_id"] == "2"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user