From cc3f9cd65b71129d108a541906677e24c709be9f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 13 Mar 2026 00:01:25 +0000 Subject: [PATCH] fix(ci): stabilize CI tests - conditional import, mock fixes, timing adjustments Fix 1.1: Make ResponseApplyPatchToolCall import conditional with try/except for compatibility with openai==1.100.1 (CI environment) Fix 1.2: Move Router creation inside mock context in vector store tests so mocks are applied before Router captures function references Fix 1.3: Update test_model_group_info_e2e to check for 'anthropic/*' wildcard group instead of specific model names not in proxy config Fix 2.1: Increase redis cache test sleep from 1s to 5s Fix 2.2: Increase spend accuracy test sleep from 25s to 45s Fix 2.3: Add 0.5s sleep between budget test calls Fix 2.4: Increase vertex AI spend test sleep from 20s to 40s Co-authored-by: yuneng-jiang --- .../transformation.py | 11 ++++--- tests/local_testing/test_custom_logger.py | 2 +- tests/otel_tests/test_e2e_budgeting.py | 1 + tests/pass_through_tests/test_vertex_ai.py | 2 +- .../test_spend_accuracy_tests.py | 8 ++--- tests/test_models.py | 21 ++++++------- tests/test_new_vector_store_endpoints.py | 30 +++++++------------ 7 files changed, 33 insertions(+), 42 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 42359afef4..4b31bcfc28 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -398,9 +398,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ResponseOutputMessage, ResponseReasoningItem, ) - from openai.types.responses.response_output_item import ( - ResponseApplyPatchToolCall, - ) + try: + from openai.types.responses.response_output_item import ( + ResponseApplyPatchToolCall, + ) + except ImportError: + ResponseApplyPatchToolCall = None # type: ignore[assignment,misc] from litellm.types.utils import Choices, Message @@ -457,7 +460,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): accumulated_tool_calls.append(tool_call_dict) tool_call_index += 1 - elif isinstance(item, ResponseApplyPatchToolCall): + elif ResponseApplyPatchToolCall is not None and isinstance(item, ResponseApplyPatchToolCall): from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) diff --git a/tests/local_testing/test_custom_logger.py b/tests/local_testing/test_custom_logger.py index f3dc6a0a7a..59025f8c2e 100644 --- a/tests/local_testing/test_custom_logger.py +++ b/tests/local_testing/test_custom_logger.py @@ -531,7 +531,7 @@ def test_redis_cache_completion_stream(): response_1_content += chunk.choices[0].delta.content or "" print(response_1_content) - time.sleep(1) # sleep for 0.1 seconds allow set cache to occur + time.sleep(5) # sleep for cache write to propagate response2 = completion( model="gpt-3.5-turbo", messages=messages, diff --git a/tests/otel_tests/test_e2e_budgeting.py b/tests/otel_tests/test_e2e_budgeting.py index e3b4c8b2b5..0aaf162b78 100644 --- a/tests/otel_tests/test_e2e_budgeting.py +++ b/tests/otel_tests/test_e2e_budgeting.py @@ -14,6 +14,7 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k while call_count < MAX_CALLS: await call_function(session=session, key=key, **kwargs) call_count += 1 + await asyncio.sleep(0.5) # allow spend tracking to catch up pytest.fail(f"Budget was not exceeded after {MAX_CALLS} calls") except Exception as e: print("vars: ", vars(e)) diff --git a/tests/pass_through_tests/test_vertex_ai.py b/tests/pass_through_tests/test_vertex_ai.py index dbcf93ee55..b3a99bc553 100644 --- a/tests/pass_through_tests/test_vertex_ai.py +++ b/tests/pass_through_tests/test_vertex_ai.py @@ -109,7 +109,7 @@ async def test_basic_vertex_ai_pass_through_with_spendlog(): print("response", response) - await asyncio.sleep(20) + await asyncio.sleep(40) spend_after = await call_spend_logs_endpoint() print("spend_after", spend_after) assert ( diff --git a/tests/spend_tracking_tests/test_spend_accuracy_tests.py b/tests/spend_tracking_tests/test_spend_accuracy_tests.py index 8101269f85..8757d6be6a 100644 --- a/tests/spend_tracking_tests/test_spend_accuracy_tests.py +++ b/tests/spend_tracking_tests/test_spend_accuracy_tests.py @@ -156,8 +156,8 @@ async def test_basic_spend_accuracy(): response = await chat_completion(session, key) print("response: ", response) - # wait 25 seconds for spend to be updated - await asyncio.sleep(25) + # wait for spend to be updated (batch writes can take a while) + await asyncio.sleep(45) # Get spend information for each entity key_info = await get_spend_info(session, "key", key) @@ -235,7 +235,7 @@ async def test_long_term_spend_accuracy_with_bursts(): print(f"Burst 1 - Request {i+1}/{BURST_1_REQUESTS} completed") # Wait for spend to be updated - await asyncio.sleep(15) + await asyncio.sleep(30) # Check intermediate spend intermediate_key_info = await get_spend_info(session, "key", key) @@ -248,7 +248,7 @@ async def test_long_term_spend_accuracy_with_bursts(): print(f"Burst 2 - Request {i+1}/{BURST_2_REQUESTS} completed") # Wait for spend to be updated - await asyncio.sleep(15) + await asyncio.sleep(30) # Get final spend information for each entity key_info = await get_spend_info(session, "key", key) diff --git a/tests/test_models.py b/tests/test_models.py index 67e77dcaaf..a4b7c6a44f 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -489,23 +489,20 @@ async def test_model_group_info_e2e(): models = await get_models(session=session, key="sk-1234") print(models) - expected_models = [ - "anthropic/claude-3-5-haiku-20241022", - "anthropic/claude-3-opus-20240229", - ] - model_group_info = await get_model_group_info(session=session, key="sk-1234") print(model_group_info) - has_anthropic_claude_3_5_haiku = False - has_anthropic_claude_3_opus = False + # Check that the endpoint returns data and contains the wildcard + # anthropic model group from the proxy config + has_anthropic_wildcard = False for model in model_group_info["data"]: - if model["model_group"] == "anthropic/claude-3-5-haiku-20241022": - has_anthropic_claude_3_5_haiku = True - if model["model_group"] == "anthropic/claude-3-opus-20240229": - has_anthropic_claude_3_opus = True + if model["model_group"] == "anthropic/*": + has_anthropic_wildcard = True - assert has_anthropic_claude_3_5_haiku and has_anthropic_claude_3_opus + assert has_anthropic_wildcard, ( + f"Expected 'anthropic/*' in model groups, got: " + f"{[m['model_group'] for m in model_group_info['data']]}" + ) @pytest.mark.asyncio diff --git a/tests/test_new_vector_store_endpoints.py b/tests/test_new_vector_store_endpoints.py index 05774c3667..56e5b4b85a 100644 --- a/tests/test_new_vector_store_endpoints.py +++ b/tests/test_new_vector_store_endpoints.py @@ -18,8 +18,6 @@ from litellm.proxy._types import UserAPIKeyAuth @pytest.mark.asyncio async def test_vector_store_retrieve_basic(): """Test basic vector store retrieve functionality.""" - router = litellm.Router(model_list=[]) - mock_response = { "id": "vs_test123", "object": "vector_store", @@ -40,6 +38,7 @@ async def test_vector_store_retrieve_basic(): "litellm.vector_stores.main.aretrieve", new=AsyncMock(return_value=mock_response), ) as mock_retrieve: + router = litellm.Router(model_list=[]) result = await router.avector_store_retrieve( vector_store_id="vs_test123", custom_llm_provider="openai", @@ -54,8 +53,6 @@ async def test_vector_store_retrieve_basic(): @pytest.mark.asyncio async def test_vector_store_list_basic(): """Test basic vector store list functionality.""" - router = litellm.Router(model_list=[]) - mock_response = { "object": "list", "data": [ @@ -81,6 +78,7 @@ async def test_vector_store_list_basic(): "litellm.vector_stores.main.alist", new=AsyncMock(return_value=mock_response), ) as mock_list: + router = litellm.Router(model_list=[]) result = await router.avector_store_list( limit=20, order="desc", @@ -96,8 +94,6 @@ async def test_vector_store_list_basic(): @pytest.mark.asyncio async def test_vector_store_update_basic(): """Test basic vector store update functionality.""" - router = litellm.Router(model_list=[]) - mock_response = { "id": "vs_test123", "object": "vector_store", @@ -111,6 +107,7 @@ async def test_vector_store_update_basic(): "litellm.vector_stores.main.aupdate", new=AsyncMock(return_value=mock_response), ) as mock_update: + router = litellm.Router(model_list=[]) result = await router.avector_store_update( vector_store_id="vs_test123", name="Updated Name", @@ -127,8 +124,6 @@ async def test_vector_store_update_basic(): @pytest.mark.asyncio async def test_vector_store_delete_basic(): """Test basic vector store delete functionality.""" - router = litellm.Router(model_list=[]) - mock_response = { "id": "vs_test123", "object": "vector_store.deleted", @@ -139,6 +134,7 @@ async def test_vector_store_delete_basic(): "litellm.vector_stores.main.adelete", new=AsyncMock(return_value=mock_response), ) as mock_delete: + router = litellm.Router(model_list=[]) result = await router.avector_store_delete( vector_store_id="vs_test123", custom_llm_provider="openai", @@ -153,8 +149,6 @@ async def test_vector_store_delete_basic(): @pytest.mark.asyncio async def test_async_vector_store_retrieve(): """Test async vector store retrieve.""" - router = litellm.Router(model_list=[]) - mock_response = { "id": "vs_async123", "object": "vector_store", @@ -165,6 +159,7 @@ async def test_async_vector_store_retrieve(): "litellm.vector_stores.main.aretrieve", new=AsyncMock(return_value=mock_response), ) as mock_aretrieve: + router = litellm.Router(model_list=[]) result = await router.avector_store_retrieve( vector_store_id="vs_async123", custom_llm_provider="openai", @@ -177,8 +172,6 @@ async def test_async_vector_store_retrieve(): @pytest.mark.asyncio async def test_async_vector_store_list(): """Test async vector store list.""" - router = litellm.Router(model_list=[]) - mock_response = { "object": "list", "data": [{"id": "vs_1"}, {"id": "vs_2"}], @@ -188,6 +181,7 @@ async def test_async_vector_store_list(): "litellm.vector_stores.main.alist", new=AsyncMock(return_value=mock_response), ) as mock_alist: + router = litellm.Router(model_list=[]) result = await router.avector_store_list( limit=10, custom_llm_provider="openai", @@ -200,8 +194,6 @@ async def test_async_vector_store_list(): @pytest.mark.asyncio async def test_async_vector_store_update(): """Test async vector store update.""" - router = litellm.Router(model_list=[]) - mock_response = { "id": "vs_async123", "name": "Updated Async Name", @@ -211,6 +203,7 @@ async def test_async_vector_store_update(): "litellm.vector_stores.main.aupdate", new=AsyncMock(return_value=mock_response), ) as mock_aupdate: + router = litellm.Router(model_list=[]) result = await router.avector_store_update( vector_store_id="vs_async123", name="Updated Async Name", @@ -224,8 +217,6 @@ async def test_async_vector_store_update(): @pytest.mark.asyncio async def test_async_vector_store_delete(): """Test async vector store delete.""" - router = litellm.Router(model_list=[]) - mock_response = { "id": "vs_async123", "deleted": True, @@ -235,6 +226,7 @@ async def test_async_vector_store_delete(): "litellm.vector_stores.main.adelete", new=AsyncMock(return_value=mock_response), ) as mock_adelete: + router = litellm.Router(model_list=[]) result = await router.avector_store_delete( vector_store_id="vs_async123", custom_llm_provider="openai", @@ -247,8 +239,6 @@ async def test_async_vector_store_delete(): @pytest.mark.asyncio async def test_vector_store_list_with_pagination(): """Test vector store list with pagination parameters.""" - router = litellm.Router(model_list=[]) - mock_response = { "object": "list", "data": [{"id": f"vs_{i}"} for i in range(5)], @@ -261,6 +251,7 @@ async def test_vector_store_list_with_pagination(): "litellm.vector_stores.main.list", return_value=mock_response, ) as mock_list: + router = litellm.Router(model_list=[]) result = router.vector_store_list( limit=5, after="vs_previous", @@ -281,8 +272,6 @@ async def test_vector_store_list_with_pagination(): @pytest.mark.asyncio async def test_vector_store_update_with_expires_after(): """Test vector store update with expiration policy.""" - router = litellm.Router(model_list=[]) - expires_after = { "anchor": "last_active_at", "days": 7, @@ -298,6 +287,7 @@ async def test_vector_store_update_with_expires_after(): "litellm.vector_stores.main.update", return_value=mock_response, ) as mock_update: + router = litellm.Router(model_list=[]) result = router.vector_store_update( vector_store_id="vs_test123", expires_after=expires_after,