test(ci): skip Fireworks tests on 404 + Gemini image-size test on 429
Four pre-existing flakes on main that gate this branch's workflow even though they're unrelated to the reasoning_effort_grid suite: 1. tests/local_testing/test_completion.py::test_completion_fireworks_ai 2. tests/local_testing/test_completion_cost.py::test_completion_cost_fireworks_ai[fireworks_ai/llama-v3p3-70b-instruct] 3. tests/llm_translation/test_fireworks_ai_translation.py::test_document_inlining_example[False] The Fireworks-hosted `llama-v3p3-70b-instruct` deployment is currently returning 404 "Model not found, inaccessible, and/or not deployed". These tests pass when the model is deployed; the issue is upstream capacity, not our code path. Wrap the live call in a try/except that pytest.skip's on litellm.NotFoundError so a Fireworks deployment hiccup no longer fails CI for unrelated PRs. 4. tests/llm_translation/test_gemini.py::test_gemini_image_size_limit_exceeded The test fetches the 32MB "Blue Marble 2002" image from Wikimedia to exercise the 50MB image-size cap. CI runners share an IP pool with noisy traffic, so Wikimedia routinely returns HTTP 429. The size-limit check never gets a chance to fire. Catch the 429 BadRequestError and pytest.skip in that case. None of these belong on this PR conceptually, but they're included per request to unblock the workflow before morning.
This commit is contained in:
parent
e29ea53c31
commit
2b00ea9ee4
@ -118,16 +118,19 @@ def test_document_inlining_example(disable_add_transform_inline_image_block):
|
||||
disable_add_transform_inline_image_block=disable_add_transform_inline_image_block,
|
||||
)
|
||||
else:
|
||||
completion = litellm.completion(
|
||||
model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "this is a test request, write a short poem",
|
||||
},
|
||||
],
|
||||
disable_add_transform_inline_image_block=disable_add_transform_inline_image_block,
|
||||
)
|
||||
try:
|
||||
completion = litellm.completion(
|
||||
model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "this is a test request, write a short poem",
|
||||
},
|
||||
],
|
||||
disable_add_transform_inline_image_block=disable_add_transform_inline_image_block,
|
||||
)
|
||||
except litellm.NotFoundError as e:
|
||||
pytest.skip(f"Fireworks model unavailable upstream (404): {e}")
|
||||
print(completion)
|
||||
|
||||
|
||||
|
||||
@ -1362,8 +1362,12 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults():
|
||||
)
|
||||
|
||||
# For Gemini 3, should not force thinkingLevel by default
|
||||
assert "thinkingLevel" not in result, "Should not force thinkingLevel for Gemini 3"
|
||||
assert "thinkingBudget" not in result, "Should NOT have thinkingBudget for Gemini 3"
|
||||
assert (
|
||||
"thinkingLevel" not in result
|
||||
), "Should not force thinkingLevel for Gemini 3"
|
||||
assert (
|
||||
"thinkingBudget" not in result
|
||||
), "Should NOT have thinkingBudget for Gemini 3"
|
||||
assert result["includeThoughts"] is True
|
||||
|
||||
# Test 2: Anthropic thinking disabled for Gemini 3
|
||||
@ -1395,7 +1399,10 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults():
|
||||
)
|
||||
|
||||
assert result_zero["includeThoughts"] is False
|
||||
assert "thinkingLevel" not in result_zero or result_zero.get("thinkingLevel") is None
|
||||
assert (
|
||||
"thinkingLevel" not in result_zero
|
||||
or result_zero.get("thinkingLevel") is None
|
||||
)
|
||||
|
||||
# Test 4: Gemini 3 flash-preview should also follow provider defaults by default
|
||||
result_gemini3flashpreview = VertexGeminiConfig._map_thinking_param(
|
||||
@ -1525,8 +1532,12 @@ def test_anthropic_thinking_param_via_map_openai_params():
|
||||
# Check that thinkingConfig was created without forced thinkingLevel
|
||||
assert "thinkingConfig" in result, "Should have thinkingConfig in optional_params"
|
||||
thinking_config = result["thinkingConfig"]
|
||||
assert "thinkingLevel" not in thinking_config, "Should not force thinkingLevel for Gemini 3 by default"
|
||||
assert "thinkingBudget" not in thinking_config, "Should NOT have thinkingBudget for Gemini 3"
|
||||
assert (
|
||||
"thinkingLevel" not in thinking_config
|
||||
), "Should not force thinkingLevel for Gemini 3 by default"
|
||||
assert (
|
||||
"thinkingBudget" not in thinking_config
|
||||
), "Should NOT have thinkingBudget for Gemini 3"
|
||||
assert thinking_config["includeThoughts"] is True
|
||||
|
||||
# Test with Gemini 2 model
|
||||
@ -1614,8 +1625,16 @@ def test_gemini_image_size_limit_exceeded():
|
||||
}
|
||||
]
|
||||
|
||||
with pytest.raises(litellm.ImageFetchError) as excinfo:
|
||||
completion(model="gemini/gemini-2.5-flash-lite", messages=messages)
|
||||
try:
|
||||
with pytest.raises(litellm.ImageFetchError) as excinfo:
|
||||
completion(model="gemini/gemini-2.5-flash-lite", messages=messages)
|
||||
except litellm.BadRequestError as e:
|
||||
# Wikimedia rate-limits CI runners (HTTP 429) for the Blue Marble
|
||||
# image, so the size-limit check never gets a chance to fire. Skip
|
||||
# rather than fail when the upstream host blocks us.
|
||||
if "429" in str(e) or "Too Many Requests" in str(e):
|
||||
pytest.skip(f"Wikimedia rate-limited the test fixture image: {e}")
|
||||
raise
|
||||
|
||||
error_message = str(excinfo.value)
|
||||
assert "Image size" in error_message
|
||||
|
||||
@ -1061,6 +1061,8 @@ def test_completion_fireworks_ai():
|
||||
messages=messages,
|
||||
)
|
||||
print(response)
|
||||
except litellm.NotFoundError as e:
|
||||
pytest.skip(f"Fireworks model unavailable upstream (404): {e}")
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
@ -1190,7 +1190,10 @@ def test_completion_cost_fireworks_ai(model):
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
messages = [{"role": "user", "content": "Hey, how's it going?"}]
|
||||
resp = litellm.completion(model=model, messages=messages) # works fine
|
||||
try:
|
||||
resp = litellm.completion(model=model, messages=messages)
|
||||
except litellm.NotFoundError as e:
|
||||
pytest.skip(f"Fireworks model unavailable upstream (404): {e}")
|
||||
|
||||
print(resp)
|
||||
cost = completion_cost(completion_response=resp)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user