ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ci): stabilize CI tests - conditional import, mock fixes, timing adjustments

Browse Source 
Fix 1.1: Make ResponseApplyPatchToolCall import conditional with try/except
  for compatibility with openai==1.100.1 (CI environment)
Fix 1.2: Move Router creation inside mock context in vector store tests
  so mocks are applied before Router captures function references
Fix 1.3: Update test_model_group_info_e2e to check for 'anthropic/*'
  wildcard group instead of specific model names not in proxy config
Fix 2.1: Increase redis cache test sleep from 1s to 5s
Fix 2.2: Increase spend accuracy test sleep from 25s to 45s
Fix 2.3: Add 0.5s sleep between budget test calls
Fix 2.4: Increase vertex AI spend test sleep from 20s to 40s

Co-authored-by: yuneng-jiang <yuneng-jiang@users.noreply.github.com>
This commit is contained in:
Cursor Agent 2026-03-13 00:01:25 +00:00
parent cb77bdaeca
commit cc3f9cd65b
No known key found for this signature in database
7 changed files with 33 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
litellm/completion_extras/litellm_responses_transformation/transformation.py
View File

@ -398,9 +398,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
ResponseOutputMessage,
ResponseReasoningItem,
)
from openai.types.responses.response_output_item import (
ResponseApplyPatchToolCall,
)
try:
from openai.types.responses.response_output_item import (
ResponseApplyPatchToolCall,
)
except ImportError:
ResponseApplyPatchToolCall = None # type: ignore[assignment,misc]
from litellm.types.utils import Choices, Message
@ -457,7 +460,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
accumulated_tool_calls.append(tool_call_dict)
tool_call_index += 1
elif isinstance(item, ResponseApplyPatchToolCall):
elif ResponseApplyPatchToolCall is not None and isinstance(item, ResponseApplyPatchToolCall):
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)

2
tests/local_testing/test_custom_logger.py
View File

@ -531,7 +531,7 @@ def test_redis_cache_completion_stream():
response_1_content += chunk.choices[0].delta.content or ""
print(response_1_content)
time.sleep(1) # sleep for 0.1 seconds allow set cache to occur
time.sleep(5) # sleep for cache write to propagate
response2 = completion(
model="gpt-3.5-turbo",
messages=messages,

1
tests/otel_tests/test_e2e_budgeting.py
View File

@ -14,6 +14,7 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k
while call_count < MAX_CALLS:
await call_function(session=session, key=key, **kwargs)
call_count += 1
await asyncio.sleep(0.5) # allow spend tracking to catch up
pytest.fail(f"Budget was not exceeded after {MAX_CALLS} calls")
except Exception as e:
print("vars: ", vars(e))

2
tests/pass_through_tests/test_vertex_ai.py
View File

@ -109,7 +109,7 @@ async def test_basic_vertex_ai_pass_through_with_spendlog():
print("response", response)
await asyncio.sleep(20)
await asyncio.sleep(40)
spend_after = await call_spend_logs_endpoint()
print("spend_after", spend_after)
assert (

8
tests/spend_tracking_tests/test_spend_accuracy_tests.py
View File

@ -156,8 +156,8 @@ async def test_basic_spend_accuracy():
response = await chat_completion(session, key)
print("response: ", response)
# wait 25 seconds for spend to be updated
await asyncio.sleep(25)
# wait for spend to be updated (batch writes can take a while)
await asyncio.sleep(45)
# Get spend information for each entity
key_info = await get_spend_info(session, "key", key)
@ -235,7 +235,7 @@ async def test_long_term_spend_accuracy_with_bursts():
print(f"Burst 1 - Request {i+1}/{BURST_1_REQUESTS} completed")
# Wait for spend to be updated
await asyncio.sleep(15)
await asyncio.sleep(30)
# Check intermediate spend
intermediate_key_info = await get_spend_info(session, "key", key)
@ -248,7 +248,7 @@ async def test_long_term_spend_accuracy_with_bursts():
print(f"Burst 2 - Request {i+1}/{BURST_2_REQUESTS} completed")
# Wait for spend to be updated
await asyncio.sleep(15)
await asyncio.sleep(30)
# Get final spend information for each entity
key_info = await get_spend_info(session, "key", key)

21
tests/test_models.py
View File

@ -489,23 +489,20 @@ async def test_model_group_info_e2e():
models = await get_models(session=session, key="sk-1234")
print(models)
expected_models = [
"anthropic/claude-3-5-haiku-20241022",
"anthropic/claude-3-opus-20240229",
]
model_group_info = await get_model_group_info(session=session, key="sk-1234")
print(model_group_info)
has_anthropic_claude_3_5_haiku = False
has_anthropic_claude_3_opus = False
# Check that the endpoint returns data and contains the wildcard
# anthropic model group from the proxy config
has_anthropic_wildcard = False
for model in model_group_info["data"]:
if model["model_group"] == "anthropic/claude-3-5-haiku-20241022":
has_anthropic_claude_3_5_haiku = True
if model["model_group"] == "anthropic/claude-3-opus-20240229":
has_anthropic_claude_3_opus = True
if model["model_group"] == "anthropic/*":
has_anthropic_wildcard = True
assert has_anthropic_claude_3_5_haiku and has_anthropic_claude_3_opus
assert has_anthropic_wildcard, (
f"Expected 'anthropic/*' in model groups, got: "
f"{[m['model_group'] for m in model_group_info['data']]}"
)
@pytest.mark.asyncio

30
tests/test_new_vector_store_endpoints.py
View File

@ -18,8 +18,6 @@ from litellm.proxy._types import UserAPIKeyAuth
@pytest.mark.asyncio
async def test_vector_store_retrieve_basic():
"""Test basic vector store retrieve functionality."""
router = litellm.Router(model_list=[])
mock_response = {
"id": "vs_test123",
"object": "vector_store",
@ -40,6 +38,7 @@ async def test_vector_store_retrieve_basic():
"litellm.vector_stores.main.aretrieve",
new=AsyncMock(return_value=mock_response),
) as mock_retrieve:
router = litellm.Router(model_list=[])
result = await router.avector_store_retrieve(
vector_store_id="vs_test123",
custom_llm_provider="openai",
@ -54,8 +53,6 @@ async def test_vector_store_retrieve_basic():
@pytest.mark.asyncio
async def test_vector_store_list_basic():
"""Test basic vector store list functionality."""
router = litellm.Router(model_list=[])
mock_response = {
"object": "list",
"data": [
@ -81,6 +78,7 @@ async def test_vector_store_list_basic():
"litellm.vector_stores.main.alist",
new=AsyncMock(return_value=mock_response),
) as mock_list:
router = litellm.Router(model_list=[])
result = await router.avector_store_list(
limit=20,
order="desc",
@ -96,8 +94,6 @@ async def test_vector_store_list_basic():
@pytest.mark.asyncio
async def test_vector_store_update_basic():
"""Test basic vector store update functionality."""
router = litellm.Router(model_list=[])
mock_response = {
"id": "vs_test123",
"object": "vector_store",
@ -111,6 +107,7 @@ async def test_vector_store_update_basic():
"litellm.vector_stores.main.aupdate",
new=AsyncMock(return_value=mock_response),
) as mock_update:
router = litellm.Router(model_list=[])
result = await router.avector_store_update(
vector_store_id="vs_test123",
name="Updated Name",
@ -127,8 +124,6 @@ async def test_vector_store_update_basic():
@pytest.mark.asyncio
async def test_vector_store_delete_basic():
"""Test basic vector store delete functionality."""
router = litellm.Router(model_list=[])
mock_response = {
"id": "vs_test123",
"object": "vector_store.deleted",
@ -139,6 +134,7 @@ async def test_vector_store_delete_basic():
"litellm.vector_stores.main.adelete",
new=AsyncMock(return_value=mock_response),
) as mock_delete:
router = litellm.Router(model_list=[])
result = await router.avector_store_delete(
vector_store_id="vs_test123",
custom_llm_provider="openai",
@ -153,8 +149,6 @@ async def test_vector_store_delete_basic():
@pytest.mark.asyncio
async def test_async_vector_store_retrieve():
"""Test async vector store retrieve."""
router = litellm.Router(model_list=[])
mock_response = {
"id": "vs_async123",
"object": "vector_store",
@ -165,6 +159,7 @@ async def test_async_vector_store_retrieve():
"litellm.vector_stores.main.aretrieve",
new=AsyncMock(return_value=mock_response),
) as mock_aretrieve:
router = litellm.Router(model_list=[])
result = await router.avector_store_retrieve(
vector_store_id="vs_async123",
custom_llm_provider="openai",
@ -177,8 +172,6 @@ async def test_async_vector_store_retrieve():
@pytest.mark.asyncio
async def test_async_vector_store_list():
"""Test async vector store list."""
router = litellm.Router(model_list=[])
mock_response = {
"object": "list",
"data": [{"id": "vs_1"}, {"id": "vs_2"}],
@ -188,6 +181,7 @@ async def test_async_vector_store_list():
"litellm.vector_stores.main.alist",
new=AsyncMock(return_value=mock_response),
) as mock_alist:
router = litellm.Router(model_list=[])
result = await router.avector_store_list(
limit=10,
custom_llm_provider="openai",
@ -200,8 +194,6 @@ async def test_async_vector_store_list():
@pytest.mark.asyncio
async def test_async_vector_store_update():
"""Test async vector store update."""
router = litellm.Router(model_list=[])
mock_response = {
"id": "vs_async123",
"name": "Updated Async Name",
@ -211,6 +203,7 @@ async def test_async_vector_store_update():
"litellm.vector_stores.main.aupdate",
new=AsyncMock(return_value=mock_response),
) as mock_aupdate:
router = litellm.Router(model_list=[])
result = await router.avector_store_update(
vector_store_id="vs_async123",
name="Updated Async Name",
@ -224,8 +217,6 @@ async def test_async_vector_store_update():
@pytest.mark.asyncio
async def test_async_vector_store_delete():
"""Test async vector store delete."""
router = litellm.Router(model_list=[])
mock_response = {
"id": "vs_async123",
"deleted": True,
@ -235,6 +226,7 @@ async def test_async_vector_store_delete():
"litellm.vector_stores.main.adelete",
new=AsyncMock(return_value=mock_response),
) as mock_adelete:
router = litellm.Router(model_list=[])
result = await router.avector_store_delete(
vector_store_id="vs_async123",
custom_llm_provider="openai",
@ -247,8 +239,6 @@ async def test_async_vector_store_delete():
@pytest.mark.asyncio
async def test_vector_store_list_with_pagination():
"""Test vector store list with pagination parameters."""
router = litellm.Router(model_list=[])
mock_response = {
"object": "list",
"data": [{"id": f"vs_{i}"} for i in range(5)],
@ -261,6 +251,7 @@ async def test_vector_store_list_with_pagination():
"litellm.vector_stores.main.list",
return_value=mock_response,
) as mock_list:
router = litellm.Router(model_list=[])
result = router.vector_store_list(
limit=5,
after="vs_previous",
@ -281,8 +272,6 @@ async def test_vector_store_list_with_pagination():
@pytest.mark.asyncio
async def test_vector_store_update_with_expires_after():
"""Test vector store update with expiration policy."""
router = litellm.Router(model_list=[])
expires_after = {
"anchor": "last_active_at",
"days": 7,
@ -298,6 +287,7 @@ async def test_vector_store_update_with_expires_after():
"litellm.vector_stores.main.update",
return_value=mock_response,
) as mock_update:
router = litellm.Router(model_list=[])
result = router.vector_store_update(
vector_store_id="vs_test123",
expires_after=expires_after,