diff --git a/litellm/constants.py b/litellm/constants.py index 32e016cd7a..fc9f5730cd 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1396,6 +1396,15 @@ LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS = int( os.getenv("LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS", 600) ) # 10 minutes default — caps the deadlock window if a pod crashes mid-rotation UI_SESSION_TOKEN_TEAM_ID = "litellm-dashboard" +LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED = os.getenv( + "LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED", "false" +) +LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS = int( + os.getenv("LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS", 86400) +) # 24 hours default +LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE = int( + os.getenv("LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE", 1000) +) LITELLM_PROXY_ADMIN_NAME = "default_user_id" ########################### CLI SSO AUTHENTICATION CONSTANTS ########################### @@ -1425,6 +1434,7 @@ CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int( ) SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup" KEY_ROTATION_JOB_NAME = "litellm_key_rotation_job" +EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME = "litellm_expired_ui_session_key_cleanup_job" SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500)) SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000)) SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100)) diff --git a/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py b/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py new file mode 100644 index 0000000000..c25d853312 --- /dev/null +++ b/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py @@ -0,0 +1,156 @@ +""" +Expired UI session key cleanup manager. + +Deletes expired virtual keys created for LiteLLM dashboard sessions. +""" + +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from litellm._logging import verbose_proxy_logger +from litellm.caching import DualCache +from litellm.constants import ( + EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME, + LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE, + LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + UI_SESSION_TOKEN_TEAM_ID, +) +from litellm.proxy._types import KeyRequest, LiteLLM_VerificationToken, UserAPIKeyAuth +from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks +from litellm.proxy.management_endpoints.key_management_endpoints import ( + delete_verification_tokens, +) +from litellm.proxy.utils import PrismaClient + + +class ExpiredUISessionKeyCleanupManager: + """ + Cleans up expired UI session keys. + """ + + def __init__( + self, + prisma_client: PrismaClient, + user_api_key_cache: DualCache, + pod_lock_manager=None, + ): + self.prisma_client = prisma_client + self.user_api_key_cache = user_api_key_cache + self.pod_lock_manager = pod_lock_manager + + async def cleanup_expired_keys(self) -> int: + """ + Main entry point for deleting expired UI session keys. + Uses PodLockManager to ensure only one pod runs cleanup in multi-pod deployments. + """ + lock_acquired = False + try: + if self.pod_lock_manager and self.pod_lock_manager.redis_cache: + lock_acquired = ( + await self.pod_lock_manager.acquire_lock( + cronjob_id=EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME, + ) + or False + ) + if not lock_acquired: + verbose_proxy_logger.debug( + "Expired UI session key cleanup: another pod is already " + "running cleanup or Redis lock acquisition failed - " + "skipping this cycle." + ) + return 0 + + verbose_proxy_logger.info("Starting expired UI session key cleanup...") + + expired_keys = await self._find_expired_ui_session_keys() + if not expired_keys: + verbose_proxy_logger.debug("No expired UI session keys found") + return 0 + + tokens = [key.token for key in expired_keys if key.token is not None] + if not tokens: + return 0 + + system_user = UserAPIKeyAuth.get_litellm_internal_jobs_user_api_key_auth() + response, keys_being_deleted = await delete_verification_tokens( + tokens=tokens, + user_api_key_cache=self.user_api_key_cache, + user_api_key_dict=system_user, + litellm_changed_by=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + ) + await KeyManagementEventHooks.async_key_deleted_hook( + data=KeyRequest(keys=tokens), + keys_being_deleted=keys_being_deleted, + response=response or {}, + user_api_key_dict=system_user, + litellm_changed_by=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + ) + deleted_count = self._get_deleted_token_count( + tokens=tokens, + response=response, + ) + verbose_proxy_logger.info( + "Deleted %s expired UI session key(s)", deleted_count + ) + return deleted_count + except Exception as e: + if getattr(e, "status_code", None) == 404: + verbose_proxy_logger.debug( + "Expired UI session key cleanup skipped because selected keys " + "were already deleted: %s", + e, + ) + return 0 + verbose_proxy_logger.error(f"Expired UI session key cleanup failed: {e}") + return 0 + finally: + if ( + lock_acquired + and self.pod_lock_manager + and self.pod_lock_manager.redis_cache + ): + await self.pod_lock_manager.release_lock( + cronjob_id=EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME, + ) + + @staticmethod + def _get_deleted_token_count( + tokens: List[str], + response: Optional[Dict[str, Any]], + ) -> int: + """ + Return the number of tokens actually deleted from the delete helper response. + """ + if response is None: + return len(tokens) + + deleted_keys = response.get("deleted_keys") + if isinstance(deleted_keys, list): + return len(deleted_keys) + if isinstance(deleted_keys, int): + return deleted_keys + if isinstance(deleted_keys, dict): + nested_deleted_keys = deleted_keys.get("deleted_keys") + if isinstance(nested_deleted_keys, list): + return len(nested_deleted_keys) + if isinstance(nested_deleted_keys, int): + return nested_deleted_keys + + failed_tokens = response.get("failed_tokens") or [] + if failed_tokens: + return max(len(tokens) - len(set(failed_tokens)), 0) + + return len(tokens) + + async def _find_expired_ui_session_keys(self) -> List[LiteLLM_VerificationToken]: + """ + Find expired LiteLLM dashboard session keys. + """ + now = datetime.now(timezone.utc) + return await self.prisma_client.db.litellm_verificationtoken.find_many( + where={ + "team_id": UI_SESSION_TOKEN_TEAM_ID, + "expires": {"lt": now}, + }, + take=LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE, + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3982d096e6..007dbe5fa7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6688,6 +6688,10 @@ class ProxyStartupEvent: Args: scheduler: The scheduler to add the background jobs to """ + global prisma_client + global proxy_logging_obj + global user_api_key_cache + ######################################################## # CloudZero Background Job ######################################################## @@ -6761,8 +6765,6 @@ class ProxyStartupEvent: ) # Get prisma_client and proxy_logging_obj from global scope - global prisma_client - global proxy_logging_obj if prisma_client is not None: # Reuse the PodLockManager from db_spend_update_writer pod_lock_manager = ( @@ -6792,6 +6794,83 @@ class ProxyStartupEvent: "Key rotation disabled (set LITELLM_KEY_ROTATION_ENABLED=true to enable)" ) + await cls._initialize_expired_ui_session_key_cleanup_background_job( + scheduler=scheduler + ) + + @classmethod + async def _initialize_expired_ui_session_key_cleanup_background_job( + cls, scheduler: AsyncIOScheduler + ): + """ + Initialize the expired UI session key cleanup background job. + """ + global prisma_client + global proxy_logging_obj + global user_api_key_cache + + ######################################################## + # Expired UI Session Key Cleanup Background Job + ######################################################## + from litellm.constants import ( + EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME, + LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED, + LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS, + ) + + expired_ui_session_key_cleanup_enabled: Optional[bool] = str_to_bool( + LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED + ) + verbose_proxy_logger.debug( + "expired_ui_session_key_cleanup_enabled: " + f"{expired_ui_session_key_cleanup_enabled}" + ) + + if expired_ui_session_key_cleanup_enabled is True: + try: + from litellm.proxy.common_utils.expired_ui_session_key_cleanup_manager import ( + ExpiredUISessionKeyCleanupManager, + ) + + if prisma_client is not None: + pod_lock_manager = ( + proxy_logging_obj.db_spend_update_writer.pod_lock_manager + ) + expired_ui_session_key_cleanup_manager = ( + ExpiredUISessionKeyCleanupManager( + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + pod_lock_manager=pod_lock_manager, + ) + ) + verbose_proxy_logger.debug( + "Expired UI session key cleanup background job scheduled " + "every " + f"{LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS} " + "seconds " + "(LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED=true)" + ) + scheduler.add_job( + expired_ui_session_key_cleanup_manager.cleanup_expired_keys, + "interval", + seconds=LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS, + id=EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME, + ) + else: + verbose_proxy_logger.warning( + "Expired UI session key cleanup enabled but prisma_client " + "not available" + ) + except Exception as e: + verbose_proxy_logger.warning( + f"Failed to setup expired UI session key cleanup job: {e}" + ) + else: + verbose_proxy_logger.debug( + "Expired UI session key cleanup disabled (set " + "LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED=true to enable)" + ) + @classmethod async def _initialize_slack_alerting_jobs( cls, diff --git a/tests/test_litellm/proxy/common_utils/test_expired_ui_session_key_cleanup_manager.py b/tests/test_litellm/proxy/common_utils/test_expired_ui_session_key_cleanup_manager.py new file mode 100644 index 0000000000..3efeeee9a2 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_expired_ui_session_key_cleanup_manager.py @@ -0,0 +1,348 @@ +""" +Test expired UI session key cleanup manager functionality. +""" + +import os +import sys +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import HTTPException, status + +sys.path.insert(0, os.path.abspath("../../../..")) + +from litellm.constants import ( + EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME, + LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE, + LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + UI_SESSION_TOKEN_TEAM_ID, +) +from litellm.proxy._types import LiteLLM_VerificationToken +from litellm.proxy.common_utils.expired_ui_session_key_cleanup_manager import ( + ExpiredUISessionKeyCleanupManager, +) + + +class TestExpiredUISessionKeyCleanupManager: + """Test the ExpiredUISessionKeyCleanupManager class functionality.""" + + @pytest.mark.asyncio + async def test_find_expired_ui_session_keys_filters_dashboard_team_and_expiry(self): + mock_prisma_client = AsyncMock() + mock_cache = MagicMock() + manager = ExpiredUISessionKeyCleanupManager( + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + ) + + now = datetime(2026, 4, 25, 12, 0, 0, tzinfo=timezone.utc) + mock_keys = [ + LiteLLM_VerificationToken( + token="expired-dashboard-token", + team_id=UI_SESSION_TOKEN_TEAM_ID, + expires=now - timedelta(seconds=1), + ) + ] + mock_prisma_client.db.litellm_verificationtoken.find_many.return_value = ( + mock_keys + ) + + with patch( + "litellm.proxy.common_utils.expired_ui_session_key_cleanup_manager.datetime" + ) as mock_datetime: + mock_datetime.now.return_value = now + mock_datetime.side_effect = lambda *args, **kwargs: datetime( + *args, **kwargs + ) + + keys = await manager._find_expired_ui_session_keys() + + mock_prisma_client.db.litellm_verificationtoken.find_many.assert_called_once_with( + where={ + "team_id": UI_SESSION_TOKEN_TEAM_ID, + "expires": {"lt": now}, + }, + take=LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE, + ) + assert keys == mock_keys + + @pytest.mark.asyncio + async def test_cleanup_expired_keys_uses_existing_delete_path(self): + mock_prisma_client = AsyncMock() + mock_cache = MagicMock() + manager = ExpiredUISessionKeyCleanupManager( + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + ) + expired_key = LiteLLM_VerificationToken( + token="expired-dashboard-token", + team_id=UI_SESSION_TOKEN_TEAM_ID, + expires=datetime.now(timezone.utc) - timedelta(seconds=1), + ) + manager._find_expired_ui_session_keys = AsyncMock(return_value=[expired_key]) + + with patch( + "litellm.proxy.common_utils.expired_ui_session_key_cleanup_manager.delete_verification_tokens", + new_callable=AsyncMock, + ) as mock_delete_verification_tokens: + mock_delete_verification_tokens.return_value = ( + {"deleted_keys": ["expired-dashboard-token"], "failed_tokens": []}, + [expired_key], + ) + with patch( + "litellm.proxy.common_utils.expired_ui_session_key_cleanup_manager.KeyManagementEventHooks.async_key_deleted_hook", + new_callable=AsyncMock, + ) as mock_key_deleted_hook: + deleted_count = await manager.cleanup_expired_keys() + + assert deleted_count == 1 + mock_delete_verification_tokens.assert_called_once() + call_kwargs = mock_delete_verification_tokens.call_args.kwargs + assert call_kwargs["tokens"] == ["expired-dashboard-token"] + assert call_kwargs["user_api_key_cache"] == mock_cache + assert ( + call_kwargs["litellm_changed_by"] + == LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME + ) + assert call_kwargs["user_api_key_dict"].user_id == "system" + mock_key_deleted_hook.assert_called_once() + hook_kwargs = mock_key_deleted_hook.call_args.kwargs + assert hook_kwargs["data"].keys == ["expired-dashboard-token"] + assert hook_kwargs["keys_being_deleted"] == [expired_key] + assert hook_kwargs["response"] == { + "deleted_keys": ["expired-dashboard-token"], + "failed_tokens": [], + } + assert ( + hook_kwargs["litellm_changed_by"] + == LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME + ) + + @pytest.mark.asyncio + async def test_cleanup_expired_keys_deletes_multiple_keys(self): + mock_prisma_client = AsyncMock() + mock_cache = MagicMock() + manager = ExpiredUISessionKeyCleanupManager( + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + ) + expired_keys = [ + LiteLLM_VerificationToken( + token="expired-dashboard-token-1", + team_id=UI_SESSION_TOKEN_TEAM_ID, + expires=datetime.now(timezone.utc) - timedelta(seconds=1), + ), + LiteLLM_VerificationToken( + token="expired-dashboard-token-2", + team_id=UI_SESSION_TOKEN_TEAM_ID, + expires=datetime.now(timezone.utc) - timedelta(seconds=1), + ), + ] + tokens = [key.token for key in expired_keys] + manager._find_expired_ui_session_keys = AsyncMock(return_value=expired_keys) + + with patch( + "litellm.proxy.common_utils.expired_ui_session_key_cleanup_manager.delete_verification_tokens", + new_callable=AsyncMock, + ) as mock_delete_verification_tokens: + mock_delete_verification_tokens.return_value = ( + {"deleted_keys": tokens, "failed_tokens": []}, + expired_keys, + ) + with patch( + "litellm.proxy.common_utils.expired_ui_session_key_cleanup_manager.KeyManagementEventHooks.async_key_deleted_hook", + new_callable=AsyncMock, + ) as mock_key_deleted_hook: + deleted_count = await manager.cleanup_expired_keys() + + assert deleted_count == 2 + assert mock_delete_verification_tokens.call_args.kwargs["tokens"] == tokens + hook_kwargs = mock_key_deleted_hook.call_args.kwargs + assert hook_kwargs["data"].keys == tokens + assert hook_kwargs["keys_being_deleted"] == expired_keys + assert hook_kwargs["response"] == {"deleted_keys": tokens, "failed_tokens": []} + + @pytest.mark.asyncio + async def test_cleanup_expired_keys_returns_successful_delete_count(self): + mock_prisma_client = AsyncMock() + mock_cache = MagicMock() + manager = ExpiredUISessionKeyCleanupManager( + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + ) + expired_keys = [ + LiteLLM_VerificationToken( + token="expired-dashboard-token-1", + team_id=UI_SESSION_TOKEN_TEAM_ID, + expires=datetime.now(timezone.utc) - timedelta(seconds=1), + ), + LiteLLM_VerificationToken( + token="expired-dashboard-token-2", + team_id=UI_SESSION_TOKEN_TEAM_ID, + expires=datetime.now(timezone.utc) - timedelta(seconds=1), + ), + ] + tokens = [key.token for key in expired_keys] + manager._find_expired_ui_session_keys = AsyncMock(return_value=expired_keys) + + with patch( + "litellm.proxy.common_utils.expired_ui_session_key_cleanup_manager.delete_verification_tokens", + new_callable=AsyncMock, + ) as mock_delete_verification_tokens: + mock_delete_verification_tokens.return_value = ( + { + "deleted_keys": ["expired-dashboard-token-1"], + "failed_tokens": ["expired-dashboard-token-2"], + }, + [expired_keys[0]], + ) + with patch( + "litellm.proxy.common_utils.expired_ui_session_key_cleanup_manager.KeyManagementEventHooks.async_key_deleted_hook", + new_callable=AsyncMock, + ): + deleted_count = await manager.cleanup_expired_keys() + + assert deleted_count == 1 + assert mock_delete_verification_tokens.call_args.kwargs["tokens"] == tokens + + @pytest.mark.asyncio + async def test_cleanup_expired_keys_counts_nested_delete_response(self): + mock_prisma_client = AsyncMock() + mock_cache = MagicMock() + manager = ExpiredUISessionKeyCleanupManager( + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + ) + expired_keys = [ + LiteLLM_VerificationToken( + token="expired-dashboard-token-1", + team_id=UI_SESSION_TOKEN_TEAM_ID, + expires=datetime.now(timezone.utc) - timedelta(seconds=1), + ), + LiteLLM_VerificationToken( + token="expired-dashboard-token-2", + team_id=UI_SESSION_TOKEN_TEAM_ID, + expires=datetime.now(timezone.utc) - timedelta(seconds=1), + ), + ] + tokens = [key.token for key in expired_keys] + manager._find_expired_ui_session_keys = AsyncMock(return_value=expired_keys) + + with patch( + "litellm.proxy.common_utils.expired_ui_session_key_cleanup_manager.delete_verification_tokens", + new_callable=AsyncMock, + ) as mock_delete_verification_tokens: + mock_delete_verification_tokens.return_value = ( + { + "deleted_keys": {"deleted_keys": 2}, + "failed_tokens": tokens, + }, + expired_keys, + ) + with patch( + "litellm.proxy.common_utils.expired_ui_session_key_cleanup_manager.KeyManagementEventHooks.async_key_deleted_hook", + new_callable=AsyncMock, + ): + deleted_count = await manager.cleanup_expired_keys() + + assert deleted_count == 2 + + @pytest.mark.asyncio + async def test_cleanup_expired_keys_treats_missing_keys_as_noop(self): + mock_prisma_client = AsyncMock() + mock_cache = MagicMock() + manager = ExpiredUISessionKeyCleanupManager( + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + ) + expired_key = LiteLLM_VerificationToken( + token="expired-dashboard-token", + team_id=UI_SESSION_TOKEN_TEAM_ID, + expires=datetime.now(timezone.utc) - timedelta(seconds=1), + ) + manager._find_expired_ui_session_keys = AsyncMock(return_value=[expired_key]) + + with patch( + "litellm.proxy.common_utils.expired_ui_session_key_cleanup_manager.delete_verification_tokens", + new_callable=AsyncMock, + ) as mock_delete_verification_tokens: + mock_delete_verification_tokens.side_effect = HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"error": "No keys found"}, + ) + with patch( + "litellm.proxy.common_utils.expired_ui_session_key_cleanup_manager.KeyManagementEventHooks.async_key_deleted_hook", + new_callable=AsyncMock, + ) as mock_key_deleted_hook: + deleted_count = await manager.cleanup_expired_keys() + + assert deleted_count == 0 + mock_key_deleted_hook.assert_not_called() + + @pytest.mark.asyncio + async def test_cleanup_expired_keys_noops_when_no_keys_found(self): + mock_prisma_client = AsyncMock() + mock_cache = MagicMock() + manager = ExpiredUISessionKeyCleanupManager( + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + ) + manager._find_expired_ui_session_keys = AsyncMock(return_value=[]) + + with patch( + "litellm.proxy.common_utils.expired_ui_session_key_cleanup_manager.delete_verification_tokens", + new_callable=AsyncMock, + ) as mock_delete_verification_tokens: + deleted_count = await manager.cleanup_expired_keys() + + assert deleted_count == 0 + mock_delete_verification_tokens.assert_not_called() + + @pytest.mark.asyncio + async def test_cleanup_expired_keys_skips_when_lock_held(self): + mock_prisma_client = AsyncMock() + mock_cache = MagicMock() + mock_pod_lock_manager = MagicMock() + mock_pod_lock_manager.redis_cache = MagicMock() + mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=False) + mock_pod_lock_manager.release_lock = AsyncMock() + + manager = ExpiredUISessionKeyCleanupManager( + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + pod_lock_manager=mock_pod_lock_manager, + ) + manager._find_expired_ui_session_keys = AsyncMock() + + deleted_count = await manager.cleanup_expired_keys() + + assert deleted_count == 0 + mock_pod_lock_manager.acquire_lock.assert_called_once_with( + cronjob_id=EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME, + ) + manager._find_expired_ui_session_keys.assert_not_called() + mock_pod_lock_manager.release_lock.assert_not_called() + + @pytest.mark.asyncio + async def test_cleanup_expired_keys_releases_acquired_lock(self): + mock_prisma_client = AsyncMock() + mock_cache = MagicMock() + mock_pod_lock_manager = MagicMock() + mock_pod_lock_manager.redis_cache = MagicMock() + mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True) + mock_pod_lock_manager.release_lock = AsyncMock() + + manager = ExpiredUISessionKeyCleanupManager( + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + pod_lock_manager=mock_pod_lock_manager, + ) + manager._find_expired_ui_session_keys = AsyncMock(return_value=[]) + + deleted_count = await manager.cleanup_expired_keys() + + assert deleted_count == 0 + mock_pod_lock_manager.release_lock.assert_called_once_with( + cronjob_id=EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME, + )