From d4830e34e58b492cfbb06a780c4710c2ee94d778 Mon Sep 17 00:00:00 2001
From: Alexsander Hamir <alexsanderhamirgomesbaptista@gmail.com>
Date: Mon, 29 Sep 2025 15:49:46 -0700
Subject: [PATCH] fix: remove router inefficiencies (from O(M*N) to O(1)) -
 62.5% faster P99 latency (#15046)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix: remove redundant deep copy

set_model_list already does the deep copy at the beginning of the call.

* fix: remove unused model_list arguments

The `model_list` parameter was being passed to classes that did not use it.

* fix: reduce per-request memory and time from O(N×M) to O(N)

No need to create a whole array for a simple look up.

* add: missing test

* fix: remove unused parameter
---
 .../proxy/anthropic_endpoints/endpoints.py    |  2 +-
 .../pass_through_endpoints.py                 |  2 +-
 litellm/proxy/route_llm_request.py            |  2 +-
 litellm/router.py                             | 32 ++++++++++++-------
 litellm/router_strategy/least_busy.py         |  5 ++-
 litellm/router_strategy/lowest_cost.py        |  3 +-
 litellm/router_strategy/lowest_latency.py     |  3 +-
 litellm/router_strategy/lowest_tpm_rpm.py     |  3 +-
 litellm/router_strategy/lowest_tpm_rpm_v2.py  |  3 +-
 .../local_testing/test_least_busy_routing.py  |  4 +--
 .../local_testing/test_lowest_cost_routing.py |  6 ++--
 .../test_lowest_latency_routing.py            | 14 ++++----
 .../local_testing/test_tpm_rpm_routing_v2.py  |  9 ++----
 .../test_router_index_management.py           | 24 ++++++++++++++
 .../test_lowest_latency_zero_tokens.py        | 20 ++----------
 15 files changed, 70 insertions(+), 62 deletions(-)

diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py
index 0dda5cecc8..e7ce5e888c 100644
--- a/litellm/proxy/anthropic_endpoints/endpoints.py
+++ b/litellm/proxy/anthropic_endpoints/endpoints.py
@@ -120,7 +120,7 @@ async def anthropic_response(  # noqa: PLR0915
         ):  # model in router deployments, calling a specific deployment on the router
             llm_coro = llm_router.aanthropic_messages(**data, specific_deployment=True)
         elif (
-            llm_router is not None and data["model"] in llm_router.get_model_ids()
+            llm_router is not None and llm_router.has_model_id(data["model"])
         ):  # model in router model list
             llm_coro = llm_router.aanthropic_messages(**data)
         elif (
diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
index c0042133b4..0eacee3b4f 100644
--- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
@@ -215,7 +215,7 @@ async def chat_completion_pass_through_endpoint(  # noqa: PLR0915
                 llm_router.aadapter_completion(**data, specific_deployment=True)
             )
         elif (
-            llm_router is not None and data["model"] in llm_router.get_model_ids()
+            llm_router is not None and llm_router.has_model_id(data["model"])
         ):  # model in router model list
             llm_response = asyncio.create_task(llm_router.aadapter_completion(**data))
         elif (
diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py
index 2a4281d635..e1a6ca2a2b 100644
--- a/litellm/proxy/route_llm_request.py
+++ b/litellm/proxy/route_llm_request.py
@@ -130,7 +130,7 @@ async def route_request(
 
         elif (
             data["model"] in router_model_names
-            or data["model"] in llm_router.get_model_ids()
+            or llm_router.has_model_id(data["model"])
         ):
             return getattr(llm_router, f"{route_type}")(**data)
 
diff --git a/litellm/router.py b/litellm/router.py
index 3cf99a4b21..0275b63698 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -415,7 +415,6 @@ class Router:
         if model_list is not None:
             # Build model index immediately to enable O(1) lookups from the start
             self._build_model_id_to_deployment_index_map(model_list)
-            model_list = copy.deepcopy(model_list)
             self.set_model_list(model_list)
             self.healthy_deployments: List = self.model_list  # type: ignore
             for m in model_list:
@@ -700,7 +699,7 @@ class Router:
             or routing_strategy == RoutingStrategy.LEAST_BUSY
         ):
             self.leastbusy_logger = LeastBusyLoggingHandler(
-                router_cache=self.cache, model_list=self.model_list
+                router_cache=self.cache
             )
             ## add callback
             if isinstance(litellm.input_callback, list):
@@ -715,7 +714,6 @@ class Router:
         ):
             self.lowesttpm_logger = LowestTPMLoggingHandler(
                 router_cache=self.cache,
-                model_list=self.model_list,
                 routing_args=routing_strategy_args,
             )
             if isinstance(litellm.callbacks, list):
@@ -726,7 +724,6 @@ class Router:
         ):
             self.lowesttpm_logger_v2 = LowestTPMLoggingHandler_v2(
                 router_cache=self.cache,
-                model_list=self.model_list,
                 routing_args=routing_strategy_args,
             )
             if isinstance(litellm.callbacks, list):
@@ -737,7 +734,6 @@ class Router:
         ):
             self.lowestlatency_logger = LowestLatencyLoggingHandler(
                 router_cache=self.cache,
-                model_list=self.model_list,
                 routing_args=routing_strategy_args,
             )
             if isinstance(litellm.callbacks, list):
@@ -748,7 +744,6 @@ class Router:
         ):
             self.lowestcost_logger = LowestCostLoggingHandler(
                 router_cache=self.cache,
-                model_list=self.model_list,
                 routing_args={},
             )
             if isinstance(litellm.callbacks, list):
@@ -972,7 +967,7 @@ class Router:
 
             ### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit)
             ## only run if model group given, not model id
-            if model not in self.get_model_ids():
+            if not self.has_model_id(model):
                 self.routing_strategy_pre_call_checks(deployment=deployment)
 
             response = litellm.completion(
@@ -5331,7 +5326,8 @@ class Router:
         """
         # check if deployment already exists
 
-        if deployment.model_info.id in self.get_model_ids():
+        _deployment_model_id = deployment.model_info.id
+        if _deployment_model_id and self.has_model_id(_deployment_model_id):
             return None
 
         # add to model list
@@ -6113,7 +6109,7 @@ class Router:
         if 'model_name' is none, returns all.
 
         Returns list of model id's.
-        """
+        """        
         ids = []
         for model in self.model_list:
             if "model_info" in model and "id" in model["model_info"]:
@@ -6126,6 +6122,19 @@ class Router:
                     ids.append(id)
         return ids
 
+    def has_model_id(self, candidate_id: str) -> bool:
+        """
+        O(1) membership check for a deployment ID without allocating large lists.
+
+        Note: Call sites may pass a variable named `model` when it actually
+        contains a deployment ID. This helper expects the deployment ID string.
+
+        Uses the existing `model_id_to_deployment_index_map` which is kept
+        in sync by `_build_model_id_to_deployment_index_map` and model-list
+        mutation helpers.
+        """
+        return candidate_id in self.model_id_to_deployment_index_map
+
     def map_team_model(self, team_model_name: str, team_id: str) -> Optional[str]:
         """
         Map a team model name to a team-specific model name.
@@ -6762,14 +6771,13 @@ class Router:
         # check if aliases set on litellm model alias map
         if specific_deployment is True:
             return model, self._get_deployment_by_litellm_model(model=model)
-        elif model in self.get_model_ids():
+        elif self.has_model_id(model):
             deployment = self.get_deployment(model_id=model)
             if deployment is not None:
                 deployment_model = deployment.litellm_params.model
                 return deployment_model, deployment.model_dump(exclude_none=True)
             raise ValueError(
-                f"LiteLLM Router: Trying to call specific deployment, but Model ID :{model} does not exist in \
-                    Model ID List: {self.get_model_ids}"
+                f"LiteLLM Router: Trying to call specific deployment, but Model ID :{model} does not exist in Model ID map"
             )
 
         _model_from_alias = self._get_model_from_alias(model=model)
diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py
index 12f3f01c83..ae0f8433d8 100644
--- a/litellm/router_strategy/least_busy.py
+++ b/litellm/router_strategy/least_busy.py
@@ -18,10 +18,9 @@ class LeastBusyLoggingHandler(CustomLogger):
     logged_success: int = 0
     logged_failure: int = 0
 
-    def __init__(self, router_cache: DualCache, model_list: list):
+    def __init__(self, router_cache: DualCache):
         self.router_cache = router_cache
-        self.mapping_deployment_to_id: dict = {}
-        self.model_list = model_list
+
 
     def log_pre_api_call(self, model, messages, kwargs):
         """
diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py
index bd28f6dc5a..b0612069df 100644
--- a/litellm/router_strategy/lowest_cost.py
+++ b/litellm/router_strategy/lowest_cost.py
@@ -16,10 +16,9 @@ class LowestCostLoggingHandler(CustomLogger):
     logged_failure: int = 0
 
     def __init__(
-        self, router_cache: DualCache, model_list: list, routing_args: dict = {}
+        self, router_cache: DualCache, routing_args: dict = {}
     ):
         self.router_cache = router_cache
-        self.model_list = model_list
 
     def log_success_event(self, kwargs, response_obj, start_time, end_time):
         try:
diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py
index 9e7ab83bf1..7492662e3b 100644
--- a/litellm/router_strategy/lowest_latency.py
+++ b/litellm/router_strategy/lowest_latency.py
@@ -32,10 +32,9 @@ class LowestLatencyLoggingHandler(CustomLogger):
     logged_failure: int = 0
 
     def __init__(
-        self, router_cache: DualCache, model_list: list, routing_args: dict = {}
+        self, router_cache: DualCache, routing_args: dict = {}
     ):
         self.router_cache = router_cache
-        self.model_list = model_list
         self.routing_args = RoutingArgs(**routing_args)
 
     def log_success_event(  # noqa: PLR0915
diff --git a/litellm/router_strategy/lowest_tpm_rpm.py b/litellm/router_strategy/lowest_tpm_rpm.py
index 735ddb3f80..e2bb0d77c4 100644
--- a/litellm/router_strategy/lowest_tpm_rpm.py
+++ b/litellm/router_strategy/lowest_tpm_rpm.py
@@ -23,10 +23,9 @@ class LowestTPMLoggingHandler(CustomLogger):
     default_cache_time_seconds: int = 1 * 60 * 60  # 1 hour
 
     def __init__(
-        self, router_cache: DualCache, model_list: list, routing_args: dict = {}
+        self, router_cache: DualCache, routing_args: dict = {}
     ):
         self.router_cache = router_cache
-        self.model_list = model_list
         self.routing_args = RoutingArgs(**routing_args)
 
     def log_success_event(self, kwargs, response_obj, start_time, end_time):
diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py
index 9e6c139314..bf3035fcc9 100644
--- a/litellm/router_strategy/lowest_tpm_rpm_v2.py
+++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py
@@ -48,10 +48,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
     default_cache_time_seconds: int = 1 * 60 * 60  # 1 hour
 
     def __init__(
-        self, router_cache: DualCache, model_list: list, routing_args: dict = {}
+        self, router_cache: DualCache, routing_args: dict = {}
     ):
         self.router_cache = router_cache
-        self.model_list = model_list
         self.routing_args = RoutingArgs(**routing_args)
         BaseRoutingStrategy.__init__(
             self,
diff --git a/tests/local_testing/test_least_busy_routing.py b/tests/local_testing/test_least_busy_routing.py
index 5a2fa19562..b30c4f3943 100644
--- a/tests/local_testing/test_least_busy_routing.py
+++ b/tests/local_testing/test_least_busy_routing.py
@@ -28,7 +28,7 @@ from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
 
 def test_model_added():
     test_cache = DualCache()
-    least_busy_logger = LeastBusyLoggingHandler(router_cache=test_cache, model_list=[])
+    least_busy_logger = LeastBusyLoggingHandler(router_cache=test_cache)
     kwargs = {
         "litellm_params": {
             "metadata": {
@@ -45,7 +45,7 @@ def test_model_added():
 
 def test_get_available_deployments():
     test_cache = DualCache()
-    least_busy_logger = LeastBusyLoggingHandler(router_cache=test_cache, model_list=[])
+    least_busy_logger = LeastBusyLoggingHandler(router_cache=test_cache)
     model_group = "gpt-3.5-turbo"
     deployment = "azure/gpt-4.1-nano"
     kwargs = {
diff --git a/tests/local_testing/test_lowest_cost_routing.py b/tests/local_testing/test_lowest_cost_routing.py
index bad8bbbb0a..3ae123e587 100644
--- a/tests/local_testing/test_lowest_cost_routing.py
+++ b/tests/local_testing/test_lowest_cost_routing.py
@@ -36,7 +36,7 @@ async def test_get_available_deployments():
         },
     ]
     lowest_cost_logger = LowestCostLoggingHandler(
-        router_cache=test_cache, model_list=model_list
+        router_cache=test_cache,
     )
     model_group = "gpt-3.5-turbo"
 
@@ -86,7 +86,7 @@ async def test_get_available_deployments_custom_price():
         },
     ]
     lowest_cost_logger = LowestCostLoggingHandler(
-        router_cache=test_cache, model_list=model_list
+        router_cache=test_cache,
     )
     model_group = "gpt-3.5-turbo"
 
@@ -187,7 +187,7 @@ async def test_get_available_endpoints_tpm_rpm_check_async(ans_rpm):
         },
     ]
     lowest_cost_logger = LowestCostLoggingHandler(
-        router_cache=test_cache, model_list=model_list
+        router_cache=test_cache
     )
     model_group = "gpt-3.5-turbo"
     d1 = [(lowest_cost_logger, "1234", 50, 0.01)] * non_ans_rpm
diff --git a/tests/local_testing/test_lowest_latency_routing.py b/tests/local_testing/test_lowest_latency_routing.py
index 2a7b0eadc4..bb2c02caca 100644
--- a/tests/local_testing/test_lowest_latency_routing.py
+++ b/tests/local_testing/test_lowest_latency_routing.py
@@ -38,9 +38,8 @@ async def test_latency_memory_leak(sync_mode):
     - make 11th call -> no change in memory
     """
     test_cache = DualCache()
-    model_list = []
     lowest_latency_logger = LowestLatencyLoggingHandler(
-        router_cache=test_cache, model_list=model_list
+        router_cache=test_cache
     )
     model_group = "gpt-3.5-turbo"
     deployment_id = "1234"
@@ -120,9 +119,8 @@ def get_size(obj, seen=None):
 
 def test_latency_updated():
     test_cache = DualCache()
-    model_list = []
     lowest_latency_logger = LowestLatencyLoggingHandler(
-        router_cache=test_cache, model_list=model_list
+        router_cache=test_cache
     )
     model_group = "gpt-3.5-turbo"
     deployment_id = "1234"
@@ -165,7 +163,7 @@ def test_latency_updated_custom_ttl():
     model_list = []
     cache_time = 3
     lowest_latency_logger = LowestLatencyLoggingHandler(
-        router_cache=test_cache, model_list=model_list, routing_args={"ttl": cache_time}
+        router_cache=test_cache, routing_args={"ttl": cache_time}
     )
     model_group = "gpt-3.5-turbo"
     deployment_id = "1234"
@@ -210,7 +208,7 @@ def test_get_available_deployments():
         },
     ]
     lowest_latency_logger = LowestLatencyLoggingHandler(
-        router_cache=test_cache, model_list=model_list
+        router_cache=test_cache
     )
     model_group = "gpt-3.5-turbo"
     ## DEPLOYMENT 1 ##
@@ -327,7 +325,7 @@ def test_get_available_endpoints_tpm_rpm_check_async(ans_rpm):
         },
     ]
     lowest_latency_logger = LowestLatencyLoggingHandler(
-        router_cache=test_cache, model_list=model_list
+        router_cache=test_cache
     )
     model_group = "gpt-3.5-turbo"
     d1 = [(lowest_latency_logger, "1234", 50, 0.01)] * non_ans_rpm
@@ -376,7 +374,7 @@ def test_get_available_endpoints_tpm_rpm_check(ans_rpm):
         },
     ]
     lowest_latency_logger = LowestLatencyLoggingHandler(
-        router_cache=test_cache, model_list=model_list
+        router_cache=test_cache
     )
     model_group = "gpt-3.5-turbo"
     ## DEPLOYMENT 1 ##
diff --git a/tests/local_testing/test_tpm_rpm_routing_v2.py b/tests/local_testing/test_tpm_rpm_routing_v2.py
index 92d2d59785..a418cd5b0e 100644
--- a/tests/local_testing/test_tpm_rpm_routing_v2.py
+++ b/tests/local_testing/test_tpm_rpm_routing_v2.py
@@ -39,9 +39,8 @@ from create_mock_standard_logging_payload import create_standard_logging_payload
 
 def test_tpm_rpm_updated():
     test_cache = DualCache()
-    model_list = []
     lowest_tpm_logger = LowestTPMLoggingHandler(
-        router_cache=test_cache, model_list=model_list
+        router_cache=test_cache
     )
     model_group = "gpt-3.5-turbo"
     deployment_id = "1234"
@@ -110,7 +109,7 @@ def test_get_available_deployments():
         },
     ]
     lowest_tpm_logger = LowestTPMLoggingHandler(
-        router_cache=test_cache, model_list=model_list
+        router_cache=test_cache
     )
     model_group = "gpt-3.5-turbo"
     ## DEPLOYMENT 1 ##
@@ -668,12 +667,10 @@ def test_return_potential_deployments():
     """
     Assert deployment at limit is filtered out
     """
-    from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2
 
     test_cache = DualCache()
-    model_list = []
     lowest_tpm_logger = LowestTPMLoggingHandler(
-        router_cache=test_cache, model_list=model_list
+        router_cache=test_cache
     )
 
     args: Dict = {
diff --git a/tests/router_unit_tests/test_router_index_management.py b/tests/router_unit_tests/test_router_index_management.py
index bdd52f66ba..ab39cc1d81 100644
--- a/tests/router_unit_tests/test_router_index_management.py
+++ b/tests/router_unit_tests/test_router_index_management.py
@@ -103,3 +103,27 @@ class TestRouterIndexManagement:
         assert router.model_id_to_deployment_index_map["id-1"] == 0
         assert router.model_id_to_deployment_index_map["id-2"] == 1
         assert router.model_id_to_deployment_index_map["id-3"] == 2
+
+    def test_has_model_id(self, router):
+        """Test has_model_id function for O(1) membership check"""
+        # Setup: Add models to router
+        router.model_list = [
+            {"model": "test1", "model_info": {"id": "model-1"}}, 
+            {"model": "test2", "model_info": {"id": "model-2"}}, 
+            {"model": "test3", "model_info": {"id": "model-3"}}
+        ]
+        router.model_id_to_deployment_index_map = {"model-1": 0, "model-2": 1, "model-3": 2}
+
+        # Test: Check existing model IDs
+        assert router.has_model_id("model-1") == True
+        assert router.has_model_id("model-2") == True
+        assert router.has_model_id("model-3") == True
+
+        # Test: Check non-existing model IDs
+        assert router.has_model_id("non-existent") == False
+        assert router.has_model_id("") == False
+        assert router.has_model_id("model-4") == False
+
+        # Test: Empty router
+        empty_router = Router(model_list=[])
+        assert empty_router.has_model_id("any-id") == False
diff --git a/tests/test_litellm/test_lowest_latency_zero_tokens.py b/tests/test_litellm/test_lowest_latency_zero_tokens.py
index 5a5209e58c..20ade6caf3 100644
--- a/tests/test_litellm/test_lowest_latency_zero_tokens.py
+++ b/tests/test_litellm/test_lowest_latency_zero_tokens.py
@@ -23,16 +23,9 @@ def test_zero_completion_tokens_no_division_error():
     (e.g., from Gemini with long contexts) caused ZeroDivisionError
     """
     test_cache = DualCache()
-    model_list = [
-        {
-            "model_name": "gemini-2.5-flash",
-            "litellm_params": {"model": "gemini/gemini-2.5-flash"},
-            "model_info": {"id": "1234"},
-        }
-    ]
-    
+
     lowest_latency_logger = LowestLatencyLoggingHandler(
-        router_cache=test_cache, model_list=model_list
+        router_cache=test_cache
     )
     
     deployment_id = "1234"
@@ -98,16 +91,9 @@ def test_zero_completion_tokens_with_time_to_first_token():
     Test that time_to_first_token calculation also handles zero completion tokens
     """
     test_cache = DualCache()
-    model_list = [
-        {
-            "model_name": "gemini-2.5-flash",
-            "litellm_params": {"model": "gemini/gemini-2.5-flash"},
-            "model_info": {"id": "1234"},
-        }
-    ]
     
     lowest_latency_logger = LowestLatencyLoggingHandler(
-        router_cache=test_cache, model_list=model_list
+        router_cache=test_cache
     )
     
     deployment_id = "1234"