ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #27415 from BerriAI/litellm_/sweet-mcclintock-2b3656

Browse Source 
[Fix] Realtime Tests: Update Deprecated OpenAI Model Pin
This commit is contained in:
yuneng-jiang 2026-05-07 15:46:51 -07:00 committed by GitHub
commit 0fb88d50dd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 55 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
tests/batches_tests/conftest.py
View File

@ -1,7 +1,6 @@
# conftest.py
import asyncio
import importlib
import os
import sys
@ -12,16 +11,6 @@ sys.path.insert(
) # Adds the parent directory to the system path
import litellm # noqa: E402,F401
from tests._vcr_conftest_common import ( # noqa: E402
VerboseReporterState,
apply_vcr_auto_marker_to_items,
record_vcr_outcome,
register_persister_if_enabled,
vcr_config_dict,
)
_verbose_state = VerboseReporterState()
@pytest.fixture(scope="session")
def event_loop():
@ -31,37 +20,3 @@ def event_loop():
loop = asyncio.new_event_loop()
yield loop
loop.close()
@pytest.fixture(scope="module")
def vcr_config():
return vcr_config_dict()
def pytest_recording_configure(config, vcr):
register_persister_if_enabled(vcr)
@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
outcome = yield
rep = outcome.get_result()
setattr(item, f"rep_{rep.when}", rep)
@pytest.fixture(autouse=True)
def _vcr_outcome_gate(request, vcr):
yield
record_vcr_outcome(request, vcr)
def pytest_configure(config):
_verbose_state.remember_pluginmanager(config)
def pytest_runtest_logreport(report):
_verbose_state.maybe_emit_verdict(report)
def pytest_collection_modifyitems(config, items):
apply_vcr_auto_marker_to_items(items)

58
tests/batches_tests/test_fine_tuning_api.py
View File

@ -47,6 +47,37 @@ class TestCustomLogger(CustomLogger):
self.standard_logging_object = kwargs["standard_logging_object"]
async def _acreate_fine_tuning_job_with_propagation_retry(
*, max_attempts: int = 12, initial_delay: float = 1.0, **kwargs
):
"""
Wrap litellm.acreate_fine_tuning_job and retry on the eventual-consistency
400 OpenAI returns when a freshly-uploaded training file isn't yet visible
to the fine-tuning endpoint (`'file-... does not exist'`).
Polling the files-retrieve endpoint or `FileObject.status` doesn't help —
OpenAI's `status` field is deprecated, and the retrieve and fine-tuning
endpoints don't share a consistency model. Retrying the operation itself
is the only reliable signal that propagation has finished.
Total budget with defaults: ~70s across 12 attempts (exp backoff capped at
8s).
"""
delay = initial_delay
last_error: Optional[openai.BadRequestError] = None
for _ in range(max_attempts):
try:
return await litellm.acreate_fine_tuning_job(**kwargs)
except openai.BadRequestError as e:
if "does not exist" not in str(e):
raise
last_error = e
await asyncio.sleep(delay)
delay = min(delay * 1.5, 8.0)
assert last_error is not None
raise last_error
@pytest.mark.asyncio
async def test_create_fine_tune_jobs_async():
try:
@ -64,9 +95,11 @@ async def test_create_fine_tune_jobs_async():
)
print("Response from creating file=", file_obj)
create_fine_tuning_response = await litellm.acreate_fine_tuning_job(
model="gpt-3.5-turbo-0125",
training_file=file_obj.id,
create_fine_tuning_response = (
await _acreate_fine_tuning_job_with_propagation_retry(
model="gpt-4o-mini-2024-07-18",
training_file=file_obj.id,
)
)
print(
@ -74,7 +107,7 @@ async def test_create_fine_tune_jobs_async():
)
assert create_fine_tuning_response.id is not None
assert create_fine_tuning_response.model == "gpt-3.5-turbo-0125"
assert create_fine_tuning_response.model == "gpt-4o-mini-2024-07-18"
await asyncio.sleep(2)
_logged_standard_logging_object = custom_logger.standard_logging_object
@ -83,7 +116,7 @@ async def test_create_fine_tune_jobs_async():
"custom_logger.standard_logging_object=",
json.dumps(_logged_standard_logging_object, indent=4),
)
assert _logged_standard_logging_object["model"] == "gpt-3.5-turbo-0125"
assert _logged_standard_logging_object["model"] == "gpt-4o-mini-2024-07-18"
assert _logged_standard_logging_object["id"] == create_fine_tuning_response.id
# list fine tuning jobs
@ -427,10 +460,10 @@ async def test_mock_openai_create_fine_tune_job():
with patch.object(client.fine_tuning.jobs, "create") as mock_create:
mock_create.return_value = FineTuningJob(
id="ft-123",
model="gpt-3.5-turbo-0125",
model="gpt-4o-mini-2024-07-18",
created_at=1677610602,
status="validating_files",
fine_tuned_model="ft:gpt-3.5-turbo-0125:org:custom_suffix:id",
fine_tuned_model="ft:gpt-4o-mini-2024-07-18:org:custom_suffix:id",
object="fine_tuning.job",
hyperparameters=Hyperparameters(
n_epochs=3,
@ -442,7 +475,7 @@ async def test_mock_openai_create_fine_tune_job():
)
response = await litellm.acreate_fine_tuning_job(
model="gpt-3.5-turbo-0125",
model="gpt-4o-mini-2024-07-18",
training_file="file-123",
hyperparameters={"n_epochs": 3},
suffix="custom_suffix",
@ -453,16 +486,19 @@ async def test_mock_openai_create_fine_tune_job():
mock_create.assert_called_once()
request_params = mock_create.call_args.kwargs
assert request_params["model"] == "gpt-3.5-turbo-0125"
assert request_params["model"] == "gpt-4o-mini-2024-07-18"
assert request_params["training_file"] == "file-123"
assert request_params["hyperparameters"] == {"n_epochs": 3}
assert request_params["suffix"] == "custom_suffix"
# Verify the response
assert response.id == "ft-123"
assert response.model == "gpt-3.5-turbo-0125"
assert response.model == "gpt-4o-mini-2024-07-18"
assert response.status == "validating_files"
assert response.fine_tuned_model == "ft:gpt-3.5-turbo-0125:org:custom_suffix:id"
assert (
response.fine_tuned_model
== "ft:gpt-4o-mini-2024-07-18:org:custom_suffix:id"
)
@pytest.mark.asyncio

18
tests/llm_translation/realtime/test_openai_realtime.py
View File

@ -101,7 +101,7 @@ async def test_openai_realtime_direct_call_no_intent():
try:
await litellm._arealtime(
model="openai/gpt-4o-realtime-preview-2024-10-01",
model="openai/gpt-4o-realtime-preview",
websocket=websocket_client,
api_key=os.environ.get("OPENAI_API_KEY"),
timeout=60,
@ -250,13 +250,13 @@ async def test_openai_realtime_direct_call_with_intent():
caught_exception = None
query_params: RealtimeQueryParams = {
"model": "openai/gpt-4o-realtime-preview-2024-10-01",
"model": "openai/gpt-4o-realtime-preview",
"intent": "chat",
}
try:
await litellm._arealtime(
model="openai/gpt-4o-realtime-preview-2024-10-01",
model="openai/gpt-4o-realtime-preview",
websocket=websocket_client,
api_key=os.environ.get("OPENAI_API_KEY"),
query_params=query_params,
@ -331,7 +331,7 @@ def test_realtime_query_params_construction():
from litellm.types.realtime import RealtimeQueryParams
# Test case 1: intent is None (should not be included)
model = "gpt-4o-realtime-preview-2024-10-01"
model = "gpt-4o-realtime-preview"
intent = None
query_params: RealtimeQueryParams = {"model": model}
@ -369,17 +369,17 @@ async def test_realtime_query_params_use_normalized_model_name(monkeypatch):
)
def fake_get_llm_provider(model, api_base=None, api_key=None):
return ("gpt-4o-realtime-preview-2024-10-01", "openai", None, None)
return ("gpt-4o-realtime-preview", "openai", None, None)
monkeypatch.setattr(realtime_main, "get_llm_provider", fake_get_llm_provider)
query_params: RealtimeQueryParams = {
"model": "openai/gpt-4o-realtime-preview-2024-10-01",
"model": "openai/gpt-4o-realtime-preview",
"intent": "chat",
}
await realtime_main._arealtime(
model="openai/gpt-4o-realtime-preview-2024-10-01",
model="openai/gpt-4o-realtime-preview",
websocket=MagicMock(),
api_key="sk-test",
query_params=query_params,
@ -387,7 +387,5 @@ async def test_realtime_query_params_use_normalized_model_name(monkeypatch):
)
called_kwargs = mock_async_realtime.call_args.kwargs
assert (
called_kwargs["query_params"]["model"] == "gpt-4o-realtime-preview-2024-10-01"
)
assert called_kwargs["query_params"]["model"] == "gpt-4o-realtime-preview"
assert called_kwargs["query_params"]["intent"] == "chat"