ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #28037 from BerriAI/litellm_/wonderful-lehmann-265845

Browse Source 
test(interactions): validate response fields against Interaction schema
This commit is contained in:
yuneng-jiang 2026-05-15 22:34:43 -07:00 committed by GitHub
commit 935a1c0eb9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 115 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
tests/llm_translation/test_fireworks_ai_translation.py
View File

@ -93,20 +93,32 @@ class TestFireworksAIAudioTranscription(BaseLLMAudioTranscriptionTest):
[True, False],
)
def test_document_inlining_example(disable_add_transform_inline_image_block):
litellm.set_verbose = True
if disable_add_transform_inline_image_block is True:
with pytest.raises(Exception):
completion = litellm.completion(
model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct",
"""
Document inlining appends ``#transform=inline`` to image/PDF URLs in the
outgoing request unless explicitly disabled. Assert the transform on the
serialized payload rather than making a live Fireworks call — the live
call only proved the model responded and broke whenever Fireworks rotated
its serverless model catalog.
"""
from unittest.mock import patch
from litellm import completion
from litellm.llms.custom_httpx.http_handler import HTTPHandler
client = HTTPHandler()
pdf_url = "https://storage.googleapis.com/fireworks-public/test/sample_resume.pdf"
with patch.object(client, "post") as mock_post:
try:
completion(
model="fireworks_ai/accounts/fireworks/models/deepseek-v3p1",
messages=[
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": "https://storage.googleapis.com/fireworks-public/test/sample_resume.pdf"
},
"image_url": {"url": pdf_url},
},
{
"type": "text",
@ -116,19 +128,19 @@ def test_document_inlining_example(disable_add_transform_inline_image_block):
}
],
disable_add_transform_inline_image_block=disable_add_transform_inline_image_block,
client=client,
)
else:
completion = litellm.completion(
model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct",
messages=[
{
"role": "user",
"content": "this is a test request, write a short poem",
},
],
disable_add_transform_inline_image_block=disable_add_transform_inline_image_block,
)
print(completion)
except Exception as e:
print(e)
mock_post.assert_called_once()
json_data = json.loads(mock_post.call_args.kwargs["data"])
sent_url = json_data["messages"][0]["content"][0]["image_url"]["url"]
if disable_add_transform_inline_image_block is True:
assert sent_url == pdf_url
assert "#transform=inline" not in sent_url
else:
assert sent_url == pdf_url + "#transform=inline"
@pytest.mark.parametrize(
@ -215,7 +227,7 @@ def test_global_disable_flag_with_transform_messages_helper(monkeypatch):
) as mock_post:
try:
completion(
model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct",
model="fireworks_ai/accounts/fireworks/models/deepseek-v3p1",
messages=[
{
"role": "user",

52
tests/local_testing/test_completion.py
View File

@ -1047,22 +1047,50 @@ def test_completion_openai_params(model):
def test_completion_fireworks_ai():
try:
litellm.set_verbose = True
messages = [
{"role": "system", "content": "You're a good bot"},
"""
Mocked so it does not depend on Fireworks' rotating serverless catalog
(no externally-verifiable model list exists). Asserts the request is
built correctly and the OpenAI-compatible response is parsed back.
"""
litellm.set_verbose = True
messages = [
{"role": "system", "content": "You're a good bot"},
{"role": "user", "content": "Hey"},
]
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.headers = {"content-type": "application/json"}
mock_response.json.return_value = {
"id": "chatcmpl-test",
"object": "chat.completion",
"created": 1234567890,
"model": "accounts/fireworks/models/deepseek-v3p1",
"choices": [
{
"role": "user",
"content": "Hey",
},
]
"index": 0,
"message": {"role": "assistant", "content": "Hello there!"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12},
}
mock_response.text = json.dumps(mock_response.json.return_value)
client = HTTPHandler()
with patch.object(client, "post", return_value=mock_response) as mock_post:
response = completion(
model="fireworks_ai/llama-v3p3-70b-instruct",
model="fireworks_ai/accounts/fireworks/models/deepseek-v3p1",
messages=messages,
client=client,
)
print(response)
except Exception as e:
pytest.fail(f"Error occurred: {e}")
mock_post.assert_called_once()
request_body = json.loads(mock_post.call_args.kwargs["data"])
assert "deepseek-v3p1" in request_body["model"]
assert request_body["messages"] == messages
assert response.choices[0].message.content == "Hello there!"
assert response.usage.total_tokens == 12
@pytest.mark.parametrize(

39
tests/local_testing/test_completion_cost.py
View File

@ -1171,7 +1171,7 @@ from litellm.llms.fireworks_ai.cost_calculator import get_base_model_for_pricing
@pytest.mark.parametrize(
"model, base_model",
[
("fireworks_ai/llama-v3p3-70b-instruct", "fireworks-ai-above-16b"),
("fireworks_ai/llama-v3p1-70b-instruct", "fireworks-ai-above-16b"),
],
)
def test_get_model_params_fireworks_ai(model, base_model):
@ -1182,18 +1182,47 @@ def test_get_model_params_fireworks_ai(model, base_model):
@pytest.mark.parametrize(
"model",
[
"fireworks_ai/llama-v3p3-70b-instruct",
"fireworks_ai/accounts/fireworks/models/deepseek-v3p1",
],
)
def test_completion_cost_fireworks_ai(model):
"""
Mocked so it does not depend on Fireworks' rotating serverless catalog.
Validates the Fireworks cost path: a parsed response with usage yields a
non-zero cost against the local cost map.
"""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
messages = [{"role": "user", "content": "Hey, how's it going?"}]
resp = litellm.completion(model=model, messages=messages) # works fine
mock_response_data = {
"id": "chatcmpl-test",
"object": "chat.completion",
"created": 1234567890,
"model": model.split("fireworks_ai/")[-1],
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "Going great, thanks!"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 8, "completion_tokens": 5, "total_tokens": 13},
}
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.headers = {"content-type": "application/json"}
mock_response.json.return_value = mock_response_data
mock_response.text = json.dumps(mock_response_data)
sync_handler = HTTPHandler()
messages = [{"role": "user", "content": "Hey, how's it going?"}]
with patch.object(HTTPHandler, "post", return_value=mock_response):
resp = litellm.completion(model=model, messages=messages, client=sync_handler)
print(resp)
cost = completion_cost(completion_response=resp)
assert cost > 0
def test_cost_azure_openai_prompt_caching():

14
tests/test_litellm/interactions/test_openapi_compliance.py
View File

@ -153,12 +153,13 @@ class TestResponseCompliance:
def test_interaction_response_fields(self, spec_dict):
"""Verify our InteractionsAPIResponse has correct fields."""
# The response is the Interaction schema
# Check CreateModelInteractionParams which includes output fields
schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"]
# The response is the dedicated `Interaction` schema. Google moved the
# output-only fields (notably the `steps` array, formerly `outputs`)
# off `CreateModelInteractionParams` and onto `Interaction`; the request
# schema no longer carries `steps`. Keep this aligned with the live spec.
schema = spec_dict["components"]["schemas"]["Interaction"]
# Output fields (readOnly). Google renamed `outputs` → `steps` in the
# upstream spec; keep this list aligned with the live schema.
# Output fields (readOnly).
output_fields = [
"id",
"status",
@ -175,7 +176,8 @@ class TestResponseCompliance:
def test_status_enum_values(self, spec_dict):
"""Verify status enum values match spec."""
schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"]
# `status` is an output-only field; validate against the response schema.
schema = spec_dict["components"]["schemas"]["Interaction"]
status_prop = schema["properties"]["status"]
# Google Interactions API uses lowercase status values (updated Feb 2026)
expected_statuses = [