fix(adaptive_router): 3 P1 review defects

- Use 'auto_router/adaptive_router' prefix in example yaml, docs, and
  README — the old 'adaptive_router/...' and 'openai/gpt-4o-mini' values
  silently skipped adaptive-router init because detection requires the
  'auto_router/adaptive_router' prefix.

- Read x-litellm-min-quality-tier from request headers (and the
  'min_quality_tier' metadata key as fallback) in async_pre_routing_hook.
  Previously the documented header was defined but never extracted, so
  the quality-floor feature was inert.

- Evict expired entries from _session_states. The cache grew without
  bound — added a parallel expiry map (same TTL as _owner_cache) and an
  opportunistic bulk sweep when the cache crosses a size threshold.

- Align adaptive-router migration SQL with Prisma schema: all count
  columns and the 'clean_credit_awarded' / 'last_processed_turn' fields
  are NOT NULL in the data model, so the migration now declares them
  NOT NULL. Fixes test_aaaasschema_migration_check.

Tests: 8 new covering header/metadata/precedence/invalid-value paths for
min_quality_tier and TTL-based eviction of _session_states.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

docs/my-website/docs/adaptive_router.md

@@ -36,7 +36,7 @@ model_list:
 
   - model_name: my-router
     litellm_params:
-      model: adaptive_router/smart-router
+      model: auto_router/adaptive_router
       adaptive_router_config:
         available_models: ["gpt-4o", "gpt-4o-mini"]
         weights:

litellm-proxy-extras/litellm_proxy_extras/migrations/20260418000000_add_adaptive_router_tables/migration.sql

@@ -16,20 +16,20 @@ CREATE TABLE "LiteLLM_AdaptiveRouterSession" (
     router_name             TEXT NOT NULL,
     model_name              TEXT NOT NULL,
     classified_type         TEXT NOT NULL,
-    misalignment_count      INTEGER DEFAULT 0,
-    stagnation_count        INTEGER DEFAULT 0,
-    disengagement_count     INTEGER DEFAULT 0,
-    satisfaction_count      INTEGER DEFAULT 0,
-    failure_count           INTEGER DEFAULT 0,
-    loop_count              INTEGER DEFAULT 0,
-    exhaustion_count        INTEGER DEFAULT 0,
+    misalignment_count      INTEGER NOT NULL DEFAULT 0,
+    stagnation_count        INTEGER NOT NULL DEFAULT 0,
+    disengagement_count     INTEGER NOT NULL DEFAULT 0,
+    satisfaction_count      INTEGER NOT NULL DEFAULT 0,
+    failure_count           INTEGER NOT NULL DEFAULT 0,
+    loop_count              INTEGER NOT NULL DEFAULT 0,
+    exhaustion_count        INTEGER NOT NULL DEFAULT 0,
     last_user_content       TEXT,
     last_assistant_content  TEXT,
-    tool_call_history       JSONB DEFAULT '[]',
-    pending_tool_calls      JSONB DEFAULT '{}',
-    turn_count              INTEGER DEFAULT 0,
-    last_processed_turn     INTEGER DEFAULT -1,
-    clean_credit_awarded    BOOLEAN DEFAULT FALSE,
+    tool_call_history       JSONB NOT NULL DEFAULT '[]',
+    pending_tool_calls      JSONB NOT NULL DEFAULT '{}',
+    turn_count              INTEGER NOT NULL DEFAULT 0,
+    last_processed_turn     INTEGER NOT NULL DEFAULT -1,
+    clean_credit_awarded    BOOLEAN NOT NULL DEFAULT FALSE,
     terminal_status         INTEGER,
     last_activity_at        TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
     PRIMARY KEY (session_id, router_name, model_name)

litellm/proxy/example_config_yaml/adaptive_router_example.yaml

@@ -17,7 +17,7 @@ model_list:
   # entries in this list).
   - model_name: smart-cheap-router
     litellm_params:
-      model: openai/gpt-4o-mini # placeholder; never actually called -- router picks from available_models
+      model: auto_router/adaptive_router # required prefix -- triggers adaptive-router init
       adaptive_router_config:
         available_models: ["fast", "smart"]
         weights:

litellm/router_strategy/adaptive_router/README.md

@@ -35,7 +35,7 @@ model_list:
 
   - model_name: smart-router
     litellm_params:
-      model: adaptive_router/smart-router
+      model: auto_router/adaptive_router
       adaptive_router_default_model: gpt-4o-mini
       adaptive_router_config:
         available_models: ["gpt-4o", "gpt-4o-mini"]

litellm/router_strategy/adaptive_router/adaptive_router.py

@@ -39,8 +39,14 @@ from litellm.router_strategy.adaptive_router.bandit import (
 from litellm.router_strategy.adaptive_router.classifier import classify_prompt
 from litellm.router_strategy.adaptive_router.config import (
     ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY,
+    MIN_QUALITY_TIER_HEADER,
+    MIN_QUALITY_TIER_METADATA_KEY,
     OWNER_CACHE_TTL_SECONDS,
 )
+
+# Sweep session-state cache when it exceeds this many live entries. Expired
+# entries are dropped in bulk; amortizes to O(1) per insert.
+_SESSION_STATE_SWEEP_THRESHOLD: int = 1024
 from litellm.router_strategy.adaptive_router.signals import (
     SessionState,
     SignalDelta,
@@ -80,6 +86,9 @@ class AdaptiveRouter:
         self._cells: Dict[Tuple[RequestType, str], BanditCell] = {}
         self._owner_cache: Dict[str, Tuple[str, float]] = {}
         self._session_states: Dict[Tuple[str, str], SessionState] = {}
+        # Parallel expiry map for _session_states, same TTL as _owner_cache.
+        # Evicted opportunistically in `get_or_create_session_state`.
+        self._session_states_expiry: Dict[Tuple[str, str], float] = {}
         self._skipped_updates_total: int = 0
         self._lock = asyncio.Lock()
 
@@ -155,7 +164,10 @@ class AdaptiveRouter:
         )
 
         request_type = classify_prompt(user_text)
-        chosen_model = await self.pick_model(request_type=request_type)
+        min_quality_tier = self._extract_min_quality_tier(request_kwargs)
+        chosen_model = await self.pick_model(
+            request_type=request_type, min_quality_tier=min_quality_tier
+        )
         verbose_router_logger.debug(
             "AdaptiveRouter[%s]: classified=%s -> chose %s",
             self.router_name,
@@ -257,6 +269,37 @@ class AdaptiveRouter:
             "queue": queue,
         }
 
+    @staticmethod
+    def _extract_min_quality_tier(
+        request_kwargs: Dict[str, Any],
+    ) -> Optional[int]:
+        """Pull `min_quality_tier` from request headers or metadata.
+
+        Precedence: headers (`x-litellm-min-quality-tier`) over metadata
+        (`min_quality_tier`). Headers arrive lowercased from the proxy but we
+        lookup case-insensitively to be safe. Unparseable values are ignored
+        (treated as "not set") rather than raising — a bad header shouldn't
+        fail the request.
+        """
+        headers = request_kwargs.get("headers") or {}
+        if isinstance(headers, dict):
+            for k, v in headers.items():
+                if isinstance(k, str) and k.lower() == MIN_QUALITY_TIER_HEADER:
+                    try:
+                        return int(v)
+                    except (TypeError, ValueError):
+                        return None
+
+        metadata = request_kwargs.get("metadata") or {}
+        if isinstance(metadata, dict):
+            raw = metadata.get(MIN_QUALITY_TIER_METADATA_KEY)
+            if raw is not None:
+                try:
+                    return int(raw)
+                except (TypeError, ValueError):
+                    return None
+        return None
+
     def _eligible_models(self, min_quality_tier: Optional[int]) -> List[str]:
         if min_quality_tier is None:
             return list(self.config.available_models)
@@ -276,6 +319,14 @@ class AdaptiveRouter:
         request_type: RequestType,
     ) -> SessionState:
         key = (session_id, model_name)
+        now = time.time()
+
+        # Opportunistic bulk sweep when the cache grows past the threshold.
+        # Cheap relative to the alternative of a bounded LRU — conversations
+        # naturally become inactive within OWNER_CACHE_TTL_SECONDS.
+        if len(self._session_states) >= _SESSION_STATE_SWEEP_THRESHOLD:
+            self._evict_expired_session_states(now)
+
         state = self._session_states.get(key)
         if state is None:
             state = SessionState(
@@ -285,8 +336,17 @@ class AdaptiveRouter:
                 classified_type=request_type.value,
             )
             self._session_states[key] = state
+        self._session_states_expiry[key] = now + OWNER_CACHE_TTL_SECONDS
         return state
 
+    def _evict_expired_session_states(self, now: float) -> None:
+        """Drop session states whose TTL has passed. O(n) but amortized O(1)
+        per insert thanks to `_SESSION_STATE_SWEEP_THRESHOLD`."""
+        expired = [k for k, exp in self._session_states_expiry.items() if exp <= now]
+        for k in expired:
+            self._session_states.pop(k, None)
+            self._session_states_expiry.pop(k, None)
+
     async def record_turn(
         self,
         session_id: str,

tests/test_litellm/router_strategy/adaptive_router/test_adaptive_router.py

@@ -222,3 +222,49 @@ async def test_load_state_from_db_handles_unknown_request_type():
     assert r._cells[(RequestType.GENERAL, "fast")].alpha == 7.0
     # Other request types kept their cold-start values.
     assert r._cells[(RequestType.WRITING, "fast")] == cold or True
+
+
+
+# ---- Session state eviction ---------------------------------------------
+
+
+def test_session_state_is_evicted_after_ttl():
+    """Entries older than OWNER_CACHE_TTL_SECONDS must be dropped when the
+    sweep runs (triggered by hitting _SESSION_STATE_SWEEP_THRESHOLD)."""
+    import time as _time
+
+    from litellm.router_strategy.adaptive_router import adaptive_router as ar
+
+    r = _make_router()
+    threshold = ar._SESSION_STATE_SWEEP_THRESHOLD
+
+    # Backdate one session so its TTL has already passed.
+    stale_key = ("sess-stale", "fast")
+    r.get_or_create_session_state("sess-stale", "fast", RequestType.GENERAL)
+    r._session_states_expiry[stale_key] = _time.time() - 1
+
+    # Fill cache up to the sweep threshold to force eviction on next insert.
+    for i in range(threshold):
+        r.get_or_create_session_state(f"sess-{i}", "fast", RequestType.GENERAL)
+
+    # Next insert triggers the sweep; stale entry should be gone.
+    r.get_or_create_session_state("sess-new", "fast", RequestType.GENERAL)
+    assert stale_key not in r._session_states
+    assert stale_key not in r._session_states_expiry
+
+
+def test_session_state_expiry_is_refreshed_on_access():
+    """Re-fetching a session state keeps it alive — TTL is a last-activity
+    timeout, not an absolute TTL."""
+    import time as _time
+
+    r = _make_router()
+    r.get_or_create_session_state("sess-A", "fast", RequestType.GENERAL)
+    first_exp = r._session_states_expiry[("sess-A", "fast")]
+
+    _time.sleep(0.01)  # move clock forward
+    r.get_or_create_session_state("sess-A", "fast", RequestType.GENERAL)
+    second_exp = r._session_states_expiry[("sess-A", "fast")]
+
+    assert second_exp > first_exp
+

tests/test_litellm/router_strategy/adaptive_router/test_async_pre_routing.py

@@ -135,3 +135,108 @@ async def test_returns_messages_unchanged_in_response():
     )
 
     assert response.messages == messages
+
+
+# ---- min_quality_tier extraction ----------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_min_quality_tier_from_header_is_forwarded_to_pick_model():
+    """`x-litellm-min-quality-tier` header should reach pick_model."""
+    r = _make_router()
+    r.pick_model = AsyncMock(return_value="smart")  # type: ignore[method-assign]
+
+    await r.async_pre_routing_hook(
+        model="smart-cheap-router",
+        request_kwargs={"headers": {"x-litellm-min-quality-tier": "3"}},
+        messages=[{"role": "user", "content": "hi"}],
+    )
+
+    assert (
+        r.pick_model.await_args.kwargs["min_quality_tier"] == 3  # type: ignore[union-attr]
+    )
+
+
+@pytest.mark.asyncio
+async def test_min_quality_tier_from_header_case_insensitive():
+    r = _make_router()
+    r.pick_model = AsyncMock(return_value="smart")  # type: ignore[method-assign]
+
+    await r.async_pre_routing_hook(
+        model="smart-cheap-router",
+        request_kwargs={"headers": {"X-LiteLLM-Min-Quality-Tier": "2"}},
+        messages=[{"role": "user", "content": "hi"}],
+    )
+
+    assert (
+        r.pick_model.await_args.kwargs["min_quality_tier"] == 2  # type: ignore[union-attr]
+    )
+
+
+@pytest.mark.asyncio
+async def test_min_quality_tier_from_metadata_key():
+    """Metadata `min_quality_tier` works when the header is absent."""
+    r = _make_router()
+    r.pick_model = AsyncMock(return_value="smart")  # type: ignore[method-assign]
+
+    await r.async_pre_routing_hook(
+        model="smart-cheap-router",
+        request_kwargs={"metadata": {"min_quality_tier": 3}},
+        messages=[{"role": "user", "content": "hi"}],
+    )
+
+    assert (
+        r.pick_model.await_args.kwargs["min_quality_tier"] == 3  # type: ignore[union-attr]
+    )
+
+
+@pytest.mark.asyncio
+async def test_header_takes_precedence_over_metadata():
+    r = _make_router()
+    r.pick_model = AsyncMock(return_value="smart")  # type: ignore[method-assign]
+
+    await r.async_pre_routing_hook(
+        model="smart-cheap-router",
+        request_kwargs={
+            "headers": {"x-litellm-min-quality-tier": "3"},
+            "metadata": {"min_quality_tier": 1},
+        },
+        messages=[{"role": "user", "content": "hi"}],
+    )
+
+    assert (
+        r.pick_model.await_args.kwargs["min_quality_tier"] == 3  # type: ignore[union-attr]
+    )
+
+
+@pytest.mark.asyncio
+async def test_missing_min_quality_tier_passes_none():
+    r = _make_router()
+    r.pick_model = AsyncMock(return_value="fast")  # type: ignore[method-assign]
+
+    await r.async_pre_routing_hook(
+        model="smart-cheap-router",
+        request_kwargs={},
+        messages=[{"role": "user", "content": "hi"}],
+    )
+
+    assert (
+        r.pick_model.await_args.kwargs["min_quality_tier"] is None  # type: ignore[union-attr]
+    )
+
+
+@pytest.mark.asyncio
+async def test_invalid_min_quality_tier_header_treated_as_none():
+    """A garbage header value must not crash the request — treat as unset."""
+    r = _make_router()
+    r.pick_model = AsyncMock(return_value="fast")  # type: ignore[method-assign]
+
+    await r.async_pre_routing_hook(
+        model="smart-cheap-router",
+        request_kwargs={"headers": {"x-litellm-min-quality-tier": "not-a-number"}},
+        messages=[{"role": "user", "content": "hi"}],
+    )
+
+    assert (
+        r.pick_model.await_args.kwargs["min_quality_tier"] is None  # type: ignore[union-attr]
+    )