test(ci): skip Fireworks tests on 404 + Gemini image-size test on 429

Four pre-existing flakes on main that gate this branch's workflow even
though they're unrelated to the reasoning_effort_grid suite:

1. tests/local_testing/test_completion.py::test_completion_fireworks_ai
2. tests/local_testing/test_completion_cost.py::test_completion_cost_fireworks_ai[fireworks_ai/llama-v3p3-70b-instruct]
3. tests/llm_translation/test_fireworks_ai_translation.py::test_document_inlining_example[False]

   The Fireworks-hosted `llama-v3p3-70b-instruct` deployment is currently
   returning 404 "Model not found, inaccessible, and/or not deployed".
   These tests pass when the model is deployed; the issue is upstream
   capacity, not our code path. Wrap the live call in a try/except that
   pytest.skip's on litellm.NotFoundError so a Fireworks deployment hiccup
   no longer fails CI for unrelated PRs.

4. tests/llm_translation/test_gemini.py::test_gemini_image_size_limit_exceeded

   The test fetches the 32MB "Blue Marble 2002" image from Wikimedia to
   exercise the 50MB image-size cap. CI runners share an IP pool with
   noisy traffic, so Wikimedia routinely returns HTTP 429. The size-limit
   check never gets a chance to fire. Catch the 429 BadRequestError and
   pytest.skip in that case.

None of these belong on this PR conceptually, but they're included per
request to unblock the workflow before morning.
This commit is contained in:
Mateo Wang 2026-05-16 07:47:25 +00:00
parent e29ea53c31
commit 2b00ea9ee4
No known key found for this signature in database
4 changed files with 45 additions and 18 deletions

View File

@ -118,16 +118,19 @@ def test_document_inlining_example(disable_add_transform_inline_image_block):
disable_add_transform_inline_image_block=disable_add_transform_inline_image_block,
)
else:
completion = litellm.completion(
model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct",
messages=[
{
"role": "user",
"content": "this is a test request, write a short poem",
},
],
disable_add_transform_inline_image_block=disable_add_transform_inline_image_block,
)
try:
completion = litellm.completion(
model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct",
messages=[
{
"role": "user",
"content": "this is a test request, write a short poem",
},
],
disable_add_transform_inline_image_block=disable_add_transform_inline_image_block,
)
except litellm.NotFoundError as e:
pytest.skip(f"Fireworks model unavailable upstream (404): {e}")
print(completion)

View File

@ -1362,8 +1362,12 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults():
)
# For Gemini 3, should not force thinkingLevel by default
assert "thinkingLevel" not in result, "Should not force thinkingLevel for Gemini 3"
assert "thinkingBudget" not in result, "Should NOT have thinkingBudget for Gemini 3"
assert (
"thinkingLevel" not in result
), "Should not force thinkingLevel for Gemini 3"
assert (
"thinkingBudget" not in result
), "Should NOT have thinkingBudget for Gemini 3"
assert result["includeThoughts"] is True
# Test 2: Anthropic thinking disabled for Gemini 3
@ -1395,7 +1399,10 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults():
)
assert result_zero["includeThoughts"] is False
assert "thinkingLevel" not in result_zero or result_zero.get("thinkingLevel") is None
assert (
"thinkingLevel" not in result_zero
or result_zero.get("thinkingLevel") is None
)
# Test 4: Gemini 3 flash-preview should also follow provider defaults by default
result_gemini3flashpreview = VertexGeminiConfig._map_thinking_param(
@ -1525,8 +1532,12 @@ def test_anthropic_thinking_param_via_map_openai_params():
# Check that thinkingConfig was created without forced thinkingLevel
assert "thinkingConfig" in result, "Should have thinkingConfig in optional_params"
thinking_config = result["thinkingConfig"]
assert "thinkingLevel" not in thinking_config, "Should not force thinkingLevel for Gemini 3 by default"
assert "thinkingBudget" not in thinking_config, "Should NOT have thinkingBudget for Gemini 3"
assert (
"thinkingLevel" not in thinking_config
), "Should not force thinkingLevel for Gemini 3 by default"
assert (
"thinkingBudget" not in thinking_config
), "Should NOT have thinkingBudget for Gemini 3"
assert thinking_config["includeThoughts"] is True
# Test with Gemini 2 model
@ -1614,8 +1625,16 @@ def test_gemini_image_size_limit_exceeded():
}
]
with pytest.raises(litellm.ImageFetchError) as excinfo:
completion(model="gemini/gemini-2.5-flash-lite", messages=messages)
try:
with pytest.raises(litellm.ImageFetchError) as excinfo:
completion(model="gemini/gemini-2.5-flash-lite", messages=messages)
except litellm.BadRequestError as e:
# Wikimedia rate-limits CI runners (HTTP 429) for the Blue Marble
# image, so the size-limit check never gets a chance to fire. Skip
# rather than fail when the upstream host blocks us.
if "429" in str(e) or "Too Many Requests" in str(e):
pytest.skip(f"Wikimedia rate-limited the test fixture image: {e}")
raise
error_message = str(excinfo.value)
assert "Image size" in error_message

View File

@ -1061,6 +1061,8 @@ def test_completion_fireworks_ai():
messages=messages,
)
print(response)
except litellm.NotFoundError as e:
pytest.skip(f"Fireworks model unavailable upstream (404): {e}")
except Exception as e:
pytest.fail(f"Error occurred: {e}")

View File

@ -1190,7 +1190,10 @@ def test_completion_cost_fireworks_ai(model):
litellm.model_cost = litellm.get_model_cost_map(url="")
messages = [{"role": "user", "content": "Hey, how's it going?"}]
resp = litellm.completion(model=model, messages=messages) # works fine
try:
resp = litellm.completion(model=model, messages=messages)
except litellm.NotFoundError as e:
pytest.skip(f"Fireworks model unavailable upstream (404): {e}")
print(resp)
cost = completion_cost(completion_response=resp)