diff --git a/tests/llm_translation/test_fireworks_ai_translation.py b/tests/llm_translation/test_fireworks_ai_translation.py index 1cc6aabdca..47b95c27ab 100644 --- a/tests/llm_translation/test_fireworks_ai_translation.py +++ b/tests/llm_translation/test_fireworks_ai_translation.py @@ -93,20 +93,32 @@ class TestFireworksAIAudioTranscription(BaseLLMAudioTranscriptionTest): [True, False], ) def test_document_inlining_example(disable_add_transform_inline_image_block): - litellm.set_verbose = True - if disable_add_transform_inline_image_block is True: - with pytest.raises(Exception): - completion = litellm.completion( - model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct", + """ + Document inlining appends ``#transform=inline`` to image/PDF URLs in the + outgoing request unless explicitly disabled. Assert the transform on the + serialized payload rather than making a live Fireworks call — the live + call only proved the model responded and broke whenever Fireworks rotated + its serverless model catalog. + """ + from unittest.mock import patch + + from litellm import completion + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + pdf_url = "https://storage.googleapis.com/fireworks-public/test/sample_resume.pdf" + + with patch.object(client, "post") as mock_post: + try: + completion( + model="fireworks_ai/accounts/fireworks/models/deepseek-v3p1", messages=[ { "role": "user", "content": [ { "type": "image_url", - "image_url": { - "url": "https://storage.googleapis.com/fireworks-public/test/sample_resume.pdf" - }, + "image_url": {"url": pdf_url}, }, { "type": "text", @@ -116,19 +128,19 @@ def test_document_inlining_example(disable_add_transform_inline_image_block): } ], disable_add_transform_inline_image_block=disable_add_transform_inline_image_block, + client=client, ) - else: - completion = litellm.completion( - model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct", - messages=[ - { - "role": "user", - "content": "this is a test request, write a short poem", - }, - ], - disable_add_transform_inline_image_block=disable_add_transform_inline_image_block, - ) - print(completion) + except Exception as e: + print(e) + + mock_post.assert_called_once() + json_data = json.loads(mock_post.call_args.kwargs["data"]) + sent_url = json_data["messages"][0]["content"][0]["image_url"]["url"] + if disable_add_transform_inline_image_block is True: + assert sent_url == pdf_url + assert "#transform=inline" not in sent_url + else: + assert sent_url == pdf_url + "#transform=inline" @pytest.mark.parametrize( @@ -215,7 +227,7 @@ def test_global_disable_flag_with_transform_messages_helper(monkeypatch): ) as mock_post: try: completion( - model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct", + model="fireworks_ai/accounts/fireworks/models/deepseek-v3p1", messages=[ { "role": "user", diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 6341fa7800..cce6d33e79 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -1047,22 +1047,50 @@ def test_completion_openai_params(model): def test_completion_fireworks_ai(): - try: - litellm.set_verbose = True - messages = [ - {"role": "system", "content": "You're a good bot"}, + """ + Mocked so it does not depend on Fireworks' rotating serverless catalog + (no externally-verifiable model list exists). Asserts the request is + built correctly and the OpenAI-compatible response is parsed back. + """ + litellm.set_verbose = True + messages = [ + {"role": "system", "content": "You're a good bot"}, + {"role": "user", "content": "Hey"}, + ] + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.json.return_value = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1234567890, + "model": "accounts/fireworks/models/deepseek-v3p1", + "choices": [ { - "role": "user", - "content": "Hey", - }, - ] + "index": 0, + "message": {"role": "assistant", "content": "Hello there!"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}, + } + mock_response.text = json.dumps(mock_response.json.return_value) + + client = HTTPHandler() + with patch.object(client, "post", return_value=mock_response) as mock_post: response = completion( - model="fireworks_ai/llama-v3p3-70b-instruct", + model="fireworks_ai/accounts/fireworks/models/deepseek-v3p1", messages=messages, + client=client, ) - print(response) - except Exception as e: - pytest.fail(f"Error occurred: {e}") + + mock_post.assert_called_once() + request_body = json.loads(mock_post.call_args.kwargs["data"]) + assert "deepseek-v3p1" in request_body["model"] + assert request_body["messages"] == messages + assert response.choices[0].message.content == "Hello there!" + assert response.usage.total_tokens == 12 @pytest.mark.parametrize( diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py index 618287e195..cf0c645615 100644 --- a/tests/local_testing/test_completion_cost.py +++ b/tests/local_testing/test_completion_cost.py @@ -1171,7 +1171,7 @@ from litellm.llms.fireworks_ai.cost_calculator import get_base_model_for_pricing @pytest.mark.parametrize( "model, base_model", [ - ("fireworks_ai/llama-v3p3-70b-instruct", "fireworks-ai-above-16b"), + ("fireworks_ai/llama-v3p1-70b-instruct", "fireworks-ai-above-16b"), ], ) def test_get_model_params_fireworks_ai(model, base_model): @@ -1182,18 +1182,47 @@ def test_get_model_params_fireworks_ai(model, base_model): @pytest.mark.parametrize( "model", [ - "fireworks_ai/llama-v3p3-70b-instruct", + "fireworks_ai/accounts/fireworks/models/deepseek-v3p1", ], ) def test_completion_cost_fireworks_ai(model): + """ + Mocked so it does not depend on Fireworks' rotating serverless catalog. + Validates the Fireworks cost path: a parsed response with usage yields a + non-zero cost against the local cost map. + """ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") - messages = [{"role": "user", "content": "Hey, how's it going?"}] - resp = litellm.completion(model=model, messages=messages) # works fine + mock_response_data = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1234567890, + "model": model.split("fireworks_ai/")[-1], + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Going great, thanks!"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 8, "completion_tokens": 5, "total_tokens": 13}, + } + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.json.return_value = mock_response_data + mock_response.text = json.dumps(mock_response_data) + + sync_handler = HTTPHandler() + messages = [{"role": "user", "content": "Hey, how's it going?"}] + + with patch.object(HTTPHandler, "post", return_value=mock_response): + resp = litellm.completion(model=model, messages=messages, client=sync_handler) - print(resp) cost = completion_cost(completion_response=resp) + assert cost > 0 def test_cost_azure_openai_prompt_caching(): diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index 11d61d4c82..1d3b6b8ae1 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -153,12 +153,13 @@ class TestResponseCompliance: def test_interaction_response_fields(self, spec_dict): """Verify our InteractionsAPIResponse has correct fields.""" - # The response is the Interaction schema - # Check CreateModelInteractionParams which includes output fields - schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"] + # The response is the dedicated `Interaction` schema. Google moved the + # output-only fields (notably the `steps` array, formerly `outputs`) + # off `CreateModelInteractionParams` and onto `Interaction`; the request + # schema no longer carries `steps`. Keep this aligned with the live spec. + schema = spec_dict["components"]["schemas"]["Interaction"] - # Output fields (readOnly). Google renamed `outputs` → `steps` in the - # upstream spec; keep this list aligned with the live schema. + # Output fields (readOnly). output_fields = [ "id", "status", @@ -175,7 +176,8 @@ class TestResponseCompliance: def test_status_enum_values(self, spec_dict): """Verify status enum values match spec.""" - schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"] + # `status` is an output-only field; validate against the response schema. + schema = spec_dict["components"]["schemas"]["Interaction"] status_prop = schema["properties"]["status"] # Google Interactions API uses lowercase status values (updated Feb 2026) expected_statuses = [