527 lines
20 KiB
Python
527 lines
20 KiB
Python
"""
|
|
Regression tests for Bedrock Anthropic models.
|
|
|
|
Tests critical functionality that has broken in the past between bedrock/invoke
|
|
and bedrock/converse routing:
|
|
1. Prompt caching support (cache_control)
|
|
2. 1M context window support (anthropic-beta header)
|
|
|
|
These tests ensure that both routing methods (invoke vs converse) maintain
|
|
feature parity and prevent regression of previously fixed issues.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, os.path.abspath("../.."))
|
|
|
|
import litellm
|
|
from litellm import completion
|
|
|
|
|
|
# Large document for caching tests (needs 1024+ tokens for Claude models)
|
|
LARGE_DOCUMENT_FOR_CACHING = """
|
|
This is a comprehensive legal agreement between Party A and Party B.
|
|
|
|
ARTICLE 1: DEFINITIONS
|
|
1.1 "Agreement" means this document and all attachments.
|
|
1.2 "Confidential Information" means any non-public information.
|
|
1.3 "Effective Date" means the date of last signature.
|
|
1.4 "Term" means the period during which this Agreement is in effect.
|
|
|
|
ARTICLE 2: SCOPE OF SERVICES
|
|
2.1 Party A agrees to provide the following services...
|
|
2.2 Party B agrees to compensate Party A for services rendered...
|
|
2.3 All services shall be performed in a professional manner...
|
|
|
|
ARTICLE 3: PAYMENT TERMS
|
|
3.1 Payment shall be made within 30 days of invoice receipt.
|
|
3.2 Late payments shall accrue interest at 1.5% per month.
|
|
3.3 All fees are non-refundable unless otherwise specified.
|
|
|
|
ARTICLE 4: INTELLECTUAL PROPERTY
|
|
4.1 All pre-existing IP remains with the original owner.
|
|
4.2 Work product created under this Agreement shall be owned by Party B.
|
|
4.3 Party A grants a license to use any tools or methodologies.
|
|
|
|
ARTICLE 5: CONFIDENTIALITY
|
|
5.1 Both parties agree to maintain confidentiality of all shared information.
|
|
5.2 Confidential information shall not be disclosed to third parties.
|
|
5.3 This obligation survives termination of the Agreement.
|
|
|
|
ARTICLE 6: TERMINATION
|
|
6.1 Either party may terminate with 30 days written notice.
|
|
6.2 Immediate termination is permitted for material breach.
|
|
6.3 Upon termination, all confidential information must be returned.
|
|
|
|
ARTICLE 7: LIMITATION OF LIABILITY
|
|
7.1 Neither party shall be liable for consequential damages.
|
|
7.2 Total liability shall not exceed fees paid in the prior 12 months.
|
|
7.3 This limitation does not apply to willful misconduct.
|
|
|
|
ARTICLE 8: DISPUTE RESOLUTION
|
|
8.1 Disputes shall first be addressed through good faith negotiation.
|
|
8.2 If negotiation fails, disputes shall be submitted to arbitration.
|
|
8.3 Arbitration shall be conducted under AAA rules.
|
|
|
|
ARTICLE 9: GENERAL PROVISIONS
|
|
9.1 This Agreement constitutes the entire understanding between parties.
|
|
9.2 Amendments must be in writing and signed by both parties.
|
|
9.3 This Agreement shall be governed by the laws of Delaware.
|
|
9.4 Neither party may assign this Agreement without consent.
|
|
9.5 Waiver of any provision shall not constitute ongoing waiver.
|
|
|
|
IN WITNESS WHEREOF, the parties have executed this Agreement.
|
|
""" * 8 # Repeat to ensure we have enough tokens (need 1024+ for Claude models)
|
|
|
|
|
|
class TestBedrockAnthropicPromptCachingRegression:
|
|
"""
|
|
Regression tests for prompt caching support across bedrock/invoke and bedrock/converse.
|
|
|
|
Issue: Prompt caching broke between invoke and converse routing due to:
|
|
- Different cache_control syntax expectations
|
|
- Incorrect beta header handling
|
|
- Missing transformation for cachePoint vs cache_control
|
|
"""
|
|
|
|
@pytest.mark.parametrize(
|
|
"model_prefix",
|
|
[
|
|
"bedrock/invoke/",
|
|
"bedrock/converse/",
|
|
],
|
|
)
|
|
def test_prompt_caching_cache_control_transforms_correctly(
|
|
self, model_prefix
|
|
):
|
|
"""
|
|
Test that cache_control in messages is correctly transformed for both invoke and converse APIs.
|
|
|
|
Regression test: Ensure cache_control works the same way for both routing methods.
|
|
- bedrock/invoke uses cache_control directly in the Anthropic Messages API format
|
|
- bedrock/converse should transform to cachePoint format
|
|
"""
|
|
from litellm.llms.bedrock.chat.converse_transformation import (
|
|
AmazonConverseConfig,
|
|
)
|
|
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
|
AmazonAnthropicClaudeConfig,
|
|
)
|
|
|
|
messages = [
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{
|
|
"type": "text",
|
|
"text": LARGE_DOCUMENT_FOR_CACHING,
|
|
"cache_control": {"type": "ephemeral"},
|
|
},
|
|
{
|
|
"type": "text",
|
|
"text": "What are the payment terms?",
|
|
},
|
|
],
|
|
},
|
|
]
|
|
|
|
if "converse" in model_prefix:
|
|
config = AmazonConverseConfig()
|
|
result = config.transform_request(
|
|
model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
|
messages=messages,
|
|
optional_params={},
|
|
litellm_params={},
|
|
headers={},
|
|
)
|
|
|
|
print(f"\n{model_prefix} Request body: {json.dumps(result, indent=2, default=str)}")
|
|
|
|
# For converse, cache_control should be transformed to cachePoint
|
|
assert "messages" in result
|
|
user_msg = result["messages"][0]
|
|
assert "content" in user_msg
|
|
|
|
# Check that cachePoint is present (Bedrock Converse format)
|
|
has_cache_point = any(
|
|
isinstance(c, dict) and "cachePoint" in c
|
|
for c in user_msg["content"]
|
|
)
|
|
# The transformation should preserve the cache marking in some form
|
|
assert "messages" in result, "messages should be present in converse request"
|
|
|
|
else:
|
|
config = AmazonAnthropicClaudeConfig()
|
|
result = config.transform_request(
|
|
model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
|
messages=messages,
|
|
optional_params={},
|
|
litellm_params={},
|
|
headers={},
|
|
)
|
|
|
|
print(f"\n{model_prefix} Request body: {json.dumps(result, indent=2, default=str)}")
|
|
|
|
# For invoke, cache_control should be preserved in messages content
|
|
assert "messages" in result
|
|
user_msg = result["messages"][0]
|
|
assert "content" in user_msg
|
|
|
|
# Check that cache_control is preserved
|
|
has_cache_control = any(
|
|
isinstance(c, dict) and "cache_control" in c
|
|
for c in user_msg["content"]
|
|
)
|
|
assert has_cache_control, "cache_control should be present in invoke messages"
|
|
|
|
@pytest.mark.parametrize(
|
|
"model_prefix",
|
|
[
|
|
"bedrock/invoke/",
|
|
"bedrock/converse/",
|
|
],
|
|
)
|
|
def test_prompt_caching_no_beta_header_added(self, model_prefix):
|
|
"""
|
|
Test that prompt-caching-2024-07-31 beta header is NOT added for Bedrock.
|
|
|
|
Regression test: Bedrock recognizes prompt caching via cache_control in the
|
|
request body, NOT through beta headers. Adding the beta header breaks requests.
|
|
|
|
This was a critical bug where litellm was incorrectly adding the Anthropic API
|
|
beta header to Bedrock requests.
|
|
"""
|
|
from litellm.llms.bedrock.chat.converse_transformation import (
|
|
AmazonConverseConfig,
|
|
)
|
|
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
|
AmazonAnthropicClaudeConfig,
|
|
)
|
|
|
|
messages = [
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{
|
|
"type": "text",
|
|
"text": "Hello",
|
|
"cache_control": {"type": "ephemeral"},
|
|
}
|
|
],
|
|
}
|
|
]
|
|
|
|
if "converse" in model_prefix:
|
|
config = AmazonConverseConfig()
|
|
result = config._transform_request_helper(
|
|
model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
|
system_content_blocks=[],
|
|
optional_params={},
|
|
messages=messages,
|
|
headers={},
|
|
)
|
|
else:
|
|
config = AmazonAnthropicClaudeConfig()
|
|
result = config.transform_request(
|
|
model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
|
messages=messages,
|
|
optional_params={},
|
|
litellm_params={},
|
|
headers={},
|
|
)
|
|
|
|
# Verify prompt-caching beta header is NOT present
|
|
if "anthropic_beta" in result:
|
|
assert "prompt-caching-2024-07-31" not in result["anthropic_beta"], (
|
|
f"{model_prefix}: prompt-caching-2024-07-31 should NOT be added as a beta header for Bedrock. "
|
|
"Bedrock recognizes prompt caching via cache_control in the request body, not beta headers."
|
|
)
|
|
|
|
# For converse, also check additionalModelRequestFields
|
|
if "converse" in model_prefix and "additionalModelRequestFields" in result:
|
|
additional_fields = result["additionalModelRequestFields"]
|
|
if "anthropic_beta" in additional_fields:
|
|
assert "prompt-caching-2024-07-31" not in additional_fields["anthropic_beta"]
|
|
|
|
|
|
class TestBedrockAnthropic1MContextRegression:
|
|
"""
|
|
Regression tests for 1M context window support across bedrock/invoke and bedrock/converse.
|
|
|
|
Issue: 1M context support broke between invoke and converse routing due to:
|
|
- Missing anthropic-beta header passthrough in converse
|
|
- Incorrect handling of context-1m-2025-08-07 beta header
|
|
"""
|
|
|
|
@pytest.mark.parametrize(
|
|
"model_prefix",
|
|
[
|
|
"bedrock/invoke/",
|
|
"bedrock/converse/",
|
|
],
|
|
)
|
|
def test_1m_context_beta_header_is_passed_via_transformation(self, model_prefix):
|
|
"""
|
|
Test that the 1M context beta header is correctly passed to Bedrock API.
|
|
|
|
Regression test: Ensure anthropic-beta: context-1m-2025-08-07 header
|
|
is correctly included in the request for both invoke and converse.
|
|
|
|
This test verifies the transformation layer directly to avoid async complexity.
|
|
"""
|
|
from litellm.llms.bedrock.chat.converse_transformation import (
|
|
AmazonConverseConfig,
|
|
)
|
|
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
|
AmazonAnthropicClaudeConfig,
|
|
)
|
|
|
|
headers = {"anthropic-beta": "context-1m-2025-08-07"}
|
|
messages = [{"role": "user", "content": "Test message"}]
|
|
|
|
if "converse" in model_prefix:
|
|
config = AmazonConverseConfig()
|
|
result = config._transform_request_helper(
|
|
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
|
system_content_blocks=[],
|
|
optional_params={},
|
|
messages=messages,
|
|
headers=headers,
|
|
)
|
|
|
|
print(f"\n{model_prefix} Request body: {json.dumps(result, indent=2, default=str)}")
|
|
|
|
# For converse, beta header should be in additionalModelRequestFields
|
|
assert "additionalModelRequestFields" in result, (
|
|
f"{model_prefix}: additionalModelRequestFields should be present for anthropic-beta headers"
|
|
)
|
|
additional_fields = result["additionalModelRequestFields"]
|
|
assert "anthropic_beta" in additional_fields, (
|
|
f"{model_prefix}: anthropic_beta should be in additionalModelRequestFields"
|
|
)
|
|
assert "context-1m-2025-08-07" in additional_fields["anthropic_beta"], (
|
|
f"{model_prefix}: context-1m-2025-08-07 should be in anthropic_beta array"
|
|
)
|
|
else:
|
|
config = AmazonAnthropicClaudeConfig()
|
|
result = config.transform_request(
|
|
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
|
messages=messages,
|
|
optional_params={},
|
|
litellm_params={},
|
|
headers=headers,
|
|
)
|
|
|
|
print(f"\n{model_prefix} Request body: {json.dumps(result, indent=2, default=str)}")
|
|
|
|
# For invoke, beta header should be in top-level request
|
|
assert "anthropic_beta" in result, (
|
|
f"{model_prefix}: anthropic_beta should be in request body"
|
|
)
|
|
assert "context-1m-2025-08-07" in result["anthropic_beta"], (
|
|
f"{model_prefix}: context-1m-2025-08-07 should be in anthropic_beta array"
|
|
)
|
|
|
|
@pytest.mark.parametrize(
|
|
"model_prefix",
|
|
[
|
|
"bedrock/invoke/",
|
|
"bedrock/converse/",
|
|
],
|
|
)
|
|
def test_1m_context_beta_header_transformation(self, model_prefix):
|
|
"""
|
|
Test that the 1M context beta header is correctly transformed at the config level.
|
|
|
|
This is a unit test that verifies the transformation logic directly without
|
|
making actual API calls.
|
|
"""
|
|
from litellm.llms.bedrock.chat.converse_transformation import (
|
|
AmazonConverseConfig,
|
|
)
|
|
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
|
AmazonAnthropicClaudeConfig,
|
|
)
|
|
|
|
headers = {"anthropic-beta": "context-1m-2025-08-07"}
|
|
messages = [{"role": "user", "content": "Test"}]
|
|
|
|
if "converse" in model_prefix:
|
|
config = AmazonConverseConfig()
|
|
result = config._transform_request_helper(
|
|
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
|
system_content_blocks=[],
|
|
optional_params={},
|
|
messages=messages,
|
|
headers=headers,
|
|
)
|
|
|
|
# Verify beta header is in additionalModelRequestFields
|
|
assert "additionalModelRequestFields" in result
|
|
additional_fields = result["additionalModelRequestFields"]
|
|
assert "anthropic_beta" in additional_fields
|
|
assert "context-1m-2025-08-07" in additional_fields["anthropic_beta"]
|
|
|
|
else:
|
|
config = AmazonAnthropicClaudeConfig()
|
|
result = config.transform_request(
|
|
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
|
messages=messages,
|
|
optional_params={},
|
|
litellm_params={},
|
|
headers=headers,
|
|
)
|
|
|
|
# Verify beta header is in top-level request
|
|
assert "anthropic_beta" in result
|
|
assert "context-1m-2025-08-07" in result["anthropic_beta"]
|
|
|
|
@pytest.mark.parametrize(
|
|
"model_prefix",
|
|
[
|
|
"bedrock/invoke/",
|
|
"bedrock/converse/",
|
|
],
|
|
)
|
|
def test_1m_context_with_multiple_beta_headers(self, model_prefix):
|
|
"""
|
|
Test that 1M context header works alongside other beta headers.
|
|
|
|
Ensures that multiple anthropic-beta values (comma-separated) are all
|
|
correctly passed through.
|
|
"""
|
|
from litellm.llms.bedrock.chat.converse_transformation import (
|
|
AmazonConverseConfig,
|
|
)
|
|
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
|
AmazonAnthropicClaudeConfig,
|
|
)
|
|
|
|
# Multiple beta headers including 1M context
|
|
headers = {
|
|
"anthropic-beta": "context-1m-2025-08-07,computer-use-2024-10-22"
|
|
}
|
|
messages = [{"role": "user", "content": "Test"}]
|
|
|
|
if "converse" in model_prefix:
|
|
config = AmazonConverseConfig()
|
|
result = config._transform_request_helper(
|
|
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
|
system_content_blocks=[],
|
|
optional_params={},
|
|
messages=messages,
|
|
headers=headers,
|
|
)
|
|
|
|
additional_fields = result["additionalModelRequestFields"]
|
|
beta_headers = additional_fields["anthropic_beta"]
|
|
|
|
else:
|
|
config = AmazonAnthropicClaudeConfig()
|
|
result = config.transform_request(
|
|
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
|
messages=messages,
|
|
optional_params={},
|
|
litellm_params={},
|
|
headers=headers,
|
|
)
|
|
|
|
beta_headers = result["anthropic_beta"]
|
|
|
|
# Verify both headers are present
|
|
assert "context-1m-2025-08-07" in beta_headers
|
|
assert "computer-use-2024-10-22" in beta_headers
|
|
|
|
|
|
class TestBedrockAnthropicCombinedRegressions:
|
|
"""
|
|
Tests that combine multiple features to ensure they work together.
|
|
"""
|
|
|
|
@pytest.mark.parametrize(
|
|
"model_prefix",
|
|
[
|
|
"bedrock/invoke/",
|
|
"bedrock/converse/",
|
|
],
|
|
)
|
|
def test_1m_context_with_prompt_caching(self, model_prefix):
|
|
"""
|
|
Test that 1M context and prompt caching work together.
|
|
|
|
This is a real-world scenario where a user might want to use both features
|
|
simultaneously.
|
|
"""
|
|
from litellm.llms.bedrock.chat.converse_transformation import (
|
|
AmazonConverseConfig,
|
|
)
|
|
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
|
AmazonAnthropicClaudeConfig,
|
|
)
|
|
|
|
headers = {"anthropic-beta": "context-1m-2025-08-07"}
|
|
messages = [
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{
|
|
"type": "text",
|
|
"text": LARGE_DOCUMENT_FOR_CACHING,
|
|
"cache_control": {"type": "ephemeral"},
|
|
},
|
|
{
|
|
"type": "text",
|
|
"text": "Summarize this document.",
|
|
},
|
|
],
|
|
}
|
|
]
|
|
|
|
if "converse" in model_prefix:
|
|
config = AmazonConverseConfig()
|
|
result = config._transform_request_helper(
|
|
model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
|
system_content_blocks=[],
|
|
optional_params={},
|
|
messages=messages,
|
|
headers=headers,
|
|
)
|
|
|
|
# Should have 1M context header
|
|
additional_fields = result["additionalModelRequestFields"]
|
|
assert "anthropic_beta" in additional_fields
|
|
assert "context-1m-2025-08-07" in additional_fields["anthropic_beta"]
|
|
|
|
# Should NOT have prompt-caching header
|
|
assert "prompt-caching-2024-07-31" not in additional_fields["anthropic_beta"]
|
|
|
|
else:
|
|
config = AmazonAnthropicClaudeConfig()
|
|
result = config.transform_request(
|
|
model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
|
messages=messages,
|
|
optional_params={},
|
|
litellm_params={},
|
|
headers=headers,
|
|
)
|
|
|
|
# Should have 1M context header
|
|
assert "anthropic_beta" in result
|
|
assert "context-1m-2025-08-07" in result["anthropic_beta"]
|
|
|
|
# Should NOT have prompt-caching header
|
|
assert "prompt-caching-2024-07-31" not in result["anthropic_beta"]
|
|
|
|
# Should have cache_control in messages
|
|
user_msg = result["messages"][0]
|
|
has_cache_control = any(
|
|
isinstance(c, dict) and "cache_control" in c
|
|
for c in user_msg["content"]
|
|
)
|
|
assert has_cache_control
|