ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #23794 from ndgigliotti/feat/bedrock-structured-output-cost-json

Browse Source 
Bedrock: move native structured output model list to cost JSON, add Sonnet 4.6
This commit is contained in:
Krrish Dholakia 2026-03-28 20:04:47 -07:00 committed by GitHub
commit 92db2df2f6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 746 additions and 1135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

361
litellm/llms/bedrock/chat/converse_transformation.py
View File

@ -91,34 +91,6 @@ UNSUPPORTED_BEDROCK_CONVERSE_BETA_PATTERNS = [
"compact-2026-01-12", # The compact beta feature is not currently supported on the Converse and ConverseStream APIs "compact-2026-01-12", # The compact beta feature is not currently supported on the Converse and ConverseStream APIs
] ]
# Models that support Bedrock's native structured outputs API (outputConfig.textFormat)
# Uses substring matching against the Bedrock model ID
# Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html
BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS = {
# Anthropic Claude 4.5+
"claude-haiku-4-5",
"claude-sonnet-4-5",
"claude-opus-4-5",
"claude-opus-4-6",
# Qwen3
"qwen3",
# DeepSeek
"deepseek-v3.1",
# Gemma 3
"gemma-3",
# MiniMax
"minimax-m2",
# Mistral (magistral-small excluded: broken constrained decoding on Bedrock)
"ministral",
"mistral-large-3",
"voxtral",
# Moonshot
"kimi-k2",
# NVIDIA
"nemotron-nano",
# OpenAI (gpt-oss excluded: broken constrained decoding, works via tool-call fallback)
}
class AmazonConverseConfig(BaseConfig): class AmazonConverseConfig(BaseConfig):
""" """
@ -188,8 +160,7 @@ class AmazonConverseConfig(BaseConfig):
if isinstance(content, list): if isinstance(content, list):
has_guarded_text = any( has_guarded_text = any(
isinstance(item, dict) and item.get("type") == "guarded_text" isinstance(item, dict) and item.get("type") == "guarded_text" for item in content
for item in content
) )
if has_guarded_text: if has_guarded_text:
continue # Skip this message if it already has guarded_text continue # Skip this message if it already has guarded_text
@ -350,13 +321,9 @@ class AmazonConverseConfig(BaseConfig):
# Check if the model is a Nova 2 model (matches nova-2-lite, nova-2-pro, etc.) # Check if the model is a Nova 2 model (matches nova-2-lite, nova-2-pro, etc.)
# Also check for nova-2/ spec prefix for imported models # Also check for nova-2/ spec prefix for imported models
return model_without_region.startswith( return model_without_region.startswith("amazon.nova-2-") or model_without_region.startswith("nova-2/")
"amazon.nova-2-"
) or model_without_region.startswith("nova-2/")
def _map_web_search_options( def _map_web_search_options(self, web_search_options: dict, model: str) -> Optional[BedrockToolBlock]:
self, web_search_options: dict, model: str
) -> Optional[BedrockToolBlock]:
""" """
Map web_search_options to Nova grounding systemTool. Map web_search_options to Nova grounding systemTool.
@ -385,9 +352,7 @@ class AmazonConverseConfig(BaseConfig):
# (unlike Anthropic), so we just enable grounding with no options # (unlike Anthropic), so we just enable grounding with no options
return BedrockToolBlock(systemTool={"name": "nova_grounding"}) return BedrockToolBlock(systemTool={"name": "nova_grounding"})
def _transform_reasoning_effort_to_reasoning_config( def _transform_reasoning_effort_to_reasoning_config(self, reasoning_effort: str) -> dict:
self, reasoning_effort: str
) -> dict:
""" """
Transform reasoning_effort parameter to Nova 2 reasoningConfig structure. Transform reasoning_effort parameter to Nova 2 reasoningConfig structure.
@ -432,9 +397,7 @@ class AmazonConverseConfig(BaseConfig):
} }
} }
def _handle_reasoning_effort_parameter( def _handle_reasoning_effort_parameter(self, model: str, reasoning_effort: str, optional_params: dict) -> None:
self, model: str, reasoning_effort: str, optional_params: dict
) -> None:
""" """
Handle the reasoning_effort parameter based on the model type. Handle the reasoning_effort parameter based on the model type.
@ -471,9 +434,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params["reasoning_effort"] = reasoning_effort optional_params["reasoning_effort"] = reasoning_effort
elif self._is_nova_2_model(model): elif self._is_nova_2_model(model):
# Nova 2 models: transform to reasoningConfig # Nova 2 models: transform to reasoningConfig
reasoning_config = self._transform_reasoning_effort_to_reasoning_config( reasoning_config = self._transform_reasoning_effort_to_reasoning_config(reasoning_effort)
reasoning_effort
)
optional_params.update(reasoning_config) optional_params.update(reasoning_config)
else: else:
# Anthropic and other models: convert to thinking parameter # Anthropic and other models: convert to thinking parameter
@ -493,8 +454,7 @@ class AmazonConverseConfig(BaseConfig):
budget = thinking.get("budget_tokens") budget = thinking.get("budget_tokens")
if isinstance(budget, int) and budget < BEDROCK_MIN_THINKING_BUDGET_TOKENS: if isinstance(budget, int) and budget < BEDROCK_MIN_THINKING_BUDGET_TOKENS:
verbose_logger.debug( verbose_logger.debug(
"Bedrock requires thinking.budget_tokens >= %d, got %d. " "Bedrock requires thinking.budget_tokens >= %d, got %d. Clamping to minimum.",
"Clamping to minimum.",
BEDROCK_MIN_THINKING_BUDGET_TOKENS, BEDROCK_MIN_THINKING_BUDGET_TOKENS,
budget, budget,
) )
@ -518,9 +478,7 @@ class AmazonConverseConfig(BaseConfig):
"parallel_tool_calls", "parallel_tool_calls",
] ]
if ( if "arn" in model: # we can't infer the model from the arn, so just add all params
"arn" in model
): # we can't infer the model from the arn, so just add all params
supported_params.append("tools") supported_params.append("tools")
supported_params.append("tool_choice") supported_params.append("tool_choice")
supported_params.append("thinking") supported_params.append("thinking")
@ -542,9 +500,7 @@ class AmazonConverseConfig(BaseConfig):
or base_model.startswith("meta.llama3-3") or base_model.startswith("meta.llama3-3")
or base_model.startswith("meta.llama4") or base_model.startswith("meta.llama4")
or base_model.startswith("amazon.nova") or base_model.startswith("amazon.nova")
or supports_function_calling( or supports_function_calling(model=model, custom_llm_provider=self.custom_llm_provider)
model=model, custom_llm_provider=self.custom_llm_provider
)
): ):
supported_params.append("tools") supported_params.append("tools")
@ -554,9 +510,7 @@ class AmazonConverseConfig(BaseConfig):
if litellm.utils.supports_tool_choice( if litellm.utils.supports_tool_choice(
model=model, custom_llm_provider=self.custom_llm_provider model=model, custom_llm_provider=self.custom_llm_provider
) or litellm.utils.supports_tool_choice( ) or litellm.utils.supports_tool_choice(model=base_model, custom_llm_provider=self.custom_llm_provider):
model=base_model, custom_llm_provider=self.custom_llm_provider
):
# only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html # only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
supported_params.append("tool_choice") supported_params.append("tool_choice")
@ -575,9 +529,7 @@ class AmazonConverseConfig(BaseConfig):
model=model, model=model,
custom_llm_provider=self.custom_llm_provider, custom_llm_provider=self.custom_llm_provider,
) )
or supports_reasoning( or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider)
model=base_model, custom_llm_provider=self.custom_llm_provider
)
): ):
supported_params.append("thinking") supported_params.append("thinking")
supported_params.append("reasoning_effort") supported_params.append("reasoning_effort")
@ -602,9 +554,7 @@ class AmazonConverseConfig(BaseConfig):
return ToolChoiceValuesBlock(auto={}) return ToolChoiceValuesBlock(auto={})
elif isinstance(tool_choice, dict): elif isinstance(tool_choice, dict):
# only supported for anthropic + mistral models - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html # only supported for anthropic + mistral models - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
specific_tool = SpecificToolChoiceBlock( specific_tool = SpecificToolChoiceBlock(name=tool_choice.get("function", {}).get("name", ""))
name=tool_choice.get("function", {}).get("name", "")
)
return ToolChoiceValuesBlock(tool=specific_tool) return ToolChoiceValuesBlock(tool=specific_tool)
else: else:
raise litellm.utils.UnsupportedParamsError( raise litellm.utils.UnsupportedParamsError(
@ -624,15 +574,9 @@ class AmazonConverseConfig(BaseConfig):
return ["mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp"] return ["mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp"]
def get_all_supported_content_types(self) -> List[str]: def get_all_supported_content_types(self) -> List[str]:
return ( return self.get_supported_image_types() + self.get_supported_document_types() + self.get_supported_video_types()
self.get_supported_image_types()
+ self.get_supported_document_types()
+ self.get_supported_video_types()
)
def is_computer_use_tool_used( def is_computer_use_tool_used(self, tools: Optional[List[OpenAIChatCompletionToolParam]], model: str) -> bool:
self, tools: Optional[List[OpenAIChatCompletionToolParam]], model: str
) -> bool:
"""Check if computer use tools are being used in the request.""" """Check if computer use tools are being used in the request."""
if tools is None: if tools is None:
return False return False
@ -645,9 +589,7 @@ class AmazonConverseConfig(BaseConfig):
return True return True
return False return False
def _transform_computer_use_tools( def _transform_computer_use_tools(self, computer_use_tools: List[OpenAIChatCompletionToolParam]) -> List[dict]:
self, computer_use_tools: List[OpenAIChatCompletionToolParam]
) -> List[dict]:
"""Transform computer use tools to Bedrock format.""" """Transform computer use tools to Bedrock format."""
transformed_tools: List[dict] = [] transformed_tools: List[dict] = []
@ -689,9 +631,7 @@ class AmazonConverseConfig(BaseConfig):
def _separate_computer_use_tools( def _separate_computer_use_tools(
self, tools: List[OpenAIChatCompletionToolParam], model: str self, tools: List[OpenAIChatCompletionToolParam], model: str
) -> Tuple[ ) -> Tuple[List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam]]:
List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam]
]:
""" """
Separate computer use tools from regular function tools. Separate computer use tools from regular function tools.
@ -763,10 +703,20 @@ class AmazonConverseConfig(BaseConfig):
return _tool return _tool
@staticmethod @staticmethod
def _supports_native_structured_outputs(model: str) -> bool: def _supports_native_structured_outputs(
"""Check if the Bedrock model supports native structured outputs (outputConfig.textFormat).""" model: str, custom_llm_provider: Optional[str] = None
return any( ) -> bool:
substring in model for substring in BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS """Check if the Bedrock model supports native structured outputs (outputConfig.textFormat).
Delegates to the standard ``supports_native_structured_output`` utility
which looks up the flag in ``litellm.model_cost`` via
``_get_model_info_helper``.
Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html
"""
from litellm.utils import supports_native_structured_output
return supports_native_structured_output(
model=model, custom_llm_provider=custom_llm_provider
) )
@staticmethod @staticmethod
@ -790,25 +740,18 @@ class AmazonConverseConfig(BaseConfig):
# Recurse into nested schemas # Recurse into nested schemas
if "properties" in result and isinstance(result["properties"], dict): if "properties" in result and isinstance(result["properties"], dict):
result["properties"] = { result["properties"] = {
k: AmazonConverseConfig._add_additional_properties_to_schema(v) k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in result["properties"].items()
for k, v in result["properties"].items()
} }
if "items" in result and isinstance(result["items"], dict): if "items" in result and isinstance(result["items"], dict):
result["items"] = AmazonConverseConfig._add_additional_properties_to_schema( result["items"] = AmazonConverseConfig._add_additional_properties_to_schema(result["items"])
result["items"]
)
for defs_key in ("$defs", "definitions"): for defs_key in ("$defs", "definitions"):
if defs_key in result and isinstance(result[defs_key], dict): if defs_key in result and isinstance(result[defs_key], dict):
result[defs_key] = { result[defs_key] = {
k: AmazonConverseConfig._add_additional_properties_to_schema(v) k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in result[defs_key].items()
for k, v in result[defs_key].items()
} }
for key in ("anyOf", "allOf", "oneOf"): for key in ("anyOf", "allOf", "oneOf"):
if key in result and isinstance(result[key], list): if key in result and isinstance(result[key], list):
result[key] = [ result[key] = [AmazonConverseConfig._add_additional_properties_to_schema(item) for item in result[key]]
AmazonConverseConfig._add_additional_properties_to_schema(item)
for item in result[key]
]
return result return result
@ -838,9 +781,7 @@ class AmazonConverseConfig(BaseConfig):
} }
""" """
if json_schema is not None: if json_schema is not None:
json_schema = AmazonConverseConfig._add_additional_properties_to_schema( json_schema = AmazonConverseConfig._add_additional_properties_to_schema(json_schema)
json_schema
)
schema_str = json.dumps(json_schema) if json_schema is not None else "{}" schema_str = json.dumps(json_schema) if json_schema is not None else "{}"
json_schema_def: JsonSchemaDefinition = {"schema": schema_str} json_schema_def: JsonSchemaDefinition = {"schema": schema_str}
if name is not None: if name is not None:
@ -862,14 +803,9 @@ class AmazonConverseConfig(BaseConfig):
non_default_params: dict, non_default_params: dict,
optional_params: dict, optional_params: dict,
): ):
optional_params = self._add_tools_to_optional_params( optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=tools)
optional_params=optional_params, tools=tools
)
if ( if "meta.llama3-3-70b-instruct-v1:0" in model and non_default_params.get("stream", False) is True:
"meta.llama3-3-70b-instruct-v1:0" in model
and non_default_params.get("stream", False) is True
):
optional_params["fake_stream"] = True optional_params["fake_stream"] = True
def map_openai_params( def map_openai_params(
@ -913,7 +849,9 @@ class AmazonConverseConfig(BaseConfig):
) )
if param == "tool_choice": if param == "tool_choice":
_tool_choice_value = self.map_tool_choice_values( _tool_choice_value = self.map_tool_choice_values(
model=model, tool_choice=value, drop_params=drop_params # type: ignore model=model,
tool_choice=value,
drop_params=drop_params, # type: ignore
) )
if _tool_choice_value is not None: if _tool_choice_value is not None:
optional_params["tool_choice"] = _tool_choice_value optional_params["tool_choice"] = _tool_choice_value
@ -1006,7 +944,7 @@ class AmazonConverseConfig(BaseConfig):
if "type" in value and value["type"] == "text": if "type" in value and value["type"] == "text":
return optional_params return optional_params
if self._supports_native_structured_outputs(model) and json_schema is not None: if self._supports_native_structured_outputs(model, self.custom_llm_provider) and json_schema is not None:
# Use Bedrock's native structured outputs API (outputConfig.textFormat) # Use Bedrock's native structured outputs API (outputConfig.textFormat)
# No synthetic tool injection, no fake_stream needed. # No synthetic tool injection, no fake_stream needed.
# Requires an explicit schema — json_object with no schema falls through # Requires an explicit schema — json_object with no schema falls through
@ -1024,14 +962,10 @@ class AmazonConverseConfig(BaseConfig):
json_schema=json_schema, json_schema=json_schema,
description=description, description=description,
) )
optional_params = self._add_tools_to_optional_params( optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool])
optional_params=optional_params, tools=[_tool]
)
if ( if (
litellm.utils.supports_tool_choice( litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider)
model=model, custom_llm_provider=self.custom_llm_provider
)
and not is_thinking_enabled and not is_thinking_enabled
): ):
optional_params["tool_choice"] = ToolChoiceValuesBlock( optional_params["tool_choice"] = ToolChoiceValuesBlock(
@ -1043,9 +977,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params["json_mode"] = True optional_params["json_mode"] = True
return optional_params return optional_params
def update_optional_params_with_thinking_tokens( def update_optional_params_with_thinking_tokens(self, non_default_params: dict, optional_params: dict):
self, non_default_params: dict, optional_params: dict
):
""" """
Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget. Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget.
@ -1063,13 +995,9 @@ class AmazonConverseConfig(BaseConfig):
is_thinking_enabled = self.is_thinking_enabled(optional_params) is_thinking_enabled = self.is_thinking_enabled(optional_params)
is_max_tokens_in_request = self.is_max_tokens_in_request(non_default_params) is_max_tokens_in_request = self.is_max_tokens_in_request(non_default_params)
if is_thinking_enabled and not is_max_tokens_in_request: if is_thinking_enabled and not is_max_tokens_in_request:
thinking_token_budget = cast(dict, optional_params["thinking"]).get( thinking_token_budget = cast(dict, optional_params["thinking"]).get("budget_tokens", None)
"budget_tokens", None
)
if thinking_token_budget is not None: if thinking_token_budget is not None:
optional_params["maxTokens"] = ( optional_params["maxTokens"] = thinking_token_budget + DEFAULT_MAX_TOKENS
thinking_token_budget + DEFAULT_MAX_TOKENS
)
@overload @overload
def _get_cache_point_block( def _get_cache_point_block(
@ -1135,23 +1063,15 @@ class AmazonConverseConfig(BaseConfig):
if message["role"] == "system": if message["role"] == "system":
system_prompt_indices.append(idx) system_prompt_indices.append(idx)
if isinstance(message["content"], str) and message["content"]: if isinstance(message["content"], str) and message["content"]:
system_content_blocks.append( system_content_blocks.append(SystemContentBlock(text=message["content"]))
SystemContentBlock(text=message["content"]) cache_block = self._get_cache_point_block(message, block_type="system", model=model)
)
cache_block = self._get_cache_point_block(
message, block_type="system", model=model
)
if cache_block: if cache_block:
system_content_blocks.append(cache_block) system_content_blocks.append(cache_block)
elif isinstance(message["content"], list): elif isinstance(message["content"], list):
for m in message["content"]: for m in message["content"]:
if m.get("type") == "text" and m.get("text"): if m.get("type") == "text" and m.get("text"):
system_content_blocks.append( system_content_blocks.append(SystemContentBlock(text=m["text"]))
SystemContentBlock(text=m["text"]) cache_block = self._get_cache_point_block(m, block_type="system", model=model)
)
cache_block = self._get_cache_point_block(
m, block_type="system", model=model
)
if cache_block: if cache_block:
system_content_blocks.append(cache_block) system_content_blocks.append(cache_block)
if len(system_prompt_indices) > 0: if len(system_prompt_indices) > 0:
@ -1189,16 +1109,10 @@ class AmazonConverseConfig(BaseConfig):
# Exceptions should not be stored in optional_params (this is a defensive fix) # Exceptions should not be stored in optional_params (this is a defensive fix)
cleaned_params = filter_exceptions_from_params(optional_params) cleaned_params = filter_exceptions_from_params(optional_params)
inference_params = safe_deep_copy(cleaned_params) inference_params = safe_deep_copy(cleaned_params)
supported_converse_params = list( supported_converse_params = list(AmazonConverseConfig.__annotations__.keys()) + ["top_k"]
AmazonConverseConfig.__annotations__.keys()
) + ["top_k"]
supported_tool_call_params = ["tools", "tool_choice"] supported_tool_call_params = ["tools", "tool_choice"]
supported_config_params = list(self.get_config_blocks().keys()) supported_config_params = list(self.get_config_blocks().keys())
total_supported_params = ( total_supported_params = supported_converse_params + supported_tool_call_params + supported_config_params
supported_converse_params
+ supported_tool_call_params
+ supported_config_params
)
inference_params.pop("json_mode", None) # used for handling json_schema inference_params.pop("json_mode", None) # used for handling json_schema
# Anthropic-only key. Bedrock expects `outputConfig` (camelCase) and # Anthropic-only key. Bedrock expects `outputConfig` (camelCase) and
# will reject `output_config` if it leaks through pass-through routes. # will reject `output_config` if it leaks through pass-through routes.
@ -1209,25 +1123,15 @@ class AmazonConverseConfig(BaseConfig):
if request_metadata is not None: if request_metadata is not None:
self._validate_request_metadata(request_metadata) self._validate_request_metadata(request_metadata)
output_config: Optional[OutputConfigBlock] = inference_params.pop( output_config: Optional[OutputConfigBlock] = inference_params.pop("outputConfig", None)
"outputConfig", None inference_params.pop("output_config", None) # Bedrock Converse doesn't support it
)
inference_params.pop(
"output_config", None
) # Bedrock Converse doesn't support it
# keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params' # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params'
additional_request_params = { additional_request_params = {k: v for k, v in inference_params.items() if k not in total_supported_params}
k: v for k, v in inference_params.items() if k not in total_supported_params inference_params = {k: v for k, v in inference_params.items() if k in total_supported_params}
}
inference_params = {
k: v for k, v in inference_params.items() if k in total_supported_params
}
# Handle parallel_tool_calls configuration # Handle parallel_tool_calls configuration
parallel_tool_use_config = additional_request_params.pop( parallel_tool_use_config = additional_request_params.pop("_parallel_tool_use_config", None)
"_parallel_tool_use_config", None
)
if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model): if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model):
for key, value in parallel_tool_use_config.items(): for key, value in parallel_tool_use_config.items():
if ( if (
@ -1242,9 +1146,7 @@ class AmazonConverseConfig(BaseConfig):
additional_request_params.pop("parallel_tool_calls", None) additional_request_params.pop("parallel_tool_calls", None)
# Only set the topK value in for models that support it # Only set the topK value in for models that support it
additional_request_params.update( additional_request_params.update(self._handle_top_k_value(model, inference_params))
self._handle_top_k_value(model, inference_params)
)
# Filter out internal/MCP-related parameters that shouldn't be sent to the API # Filter out internal/MCP-related parameters that shouldn't be sent to the API
# These are LiteLLM internal parameters, not API parameters # These are LiteLLM internal parameters, not API parameters
@ -1253,9 +1155,7 @@ class AmazonConverseConfig(BaseConfig):
# Filter out non-serializable objects (exceptions, callables, logging objects, etc.) # Filter out non-serializable objects (exceptions, callables, logging objects, etc.)
# from additional_request_params to prevent JSON serialization errors # from additional_request_params to prevent JSON serialization errors
# This filters: Exception objects, callable objects (functions), Logging objects, etc. # This filters: Exception objects, callable objects (functions), Logging objects, etc.
additional_request_params = filter_exceptions_from_params( additional_request_params = filter_exceptions_from_params(additional_request_params)
additional_request_params
)
return ( return (
inference_params, inference_params,
@ -1302,9 +1202,7 @@ class AmazonConverseConfig(BaseConfig):
# Only separate tools if computer use tools are actually present # Only separate tools if computer use tools are actually present
if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model): if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model):
# Separate computer use tools from regular function tools # Separate computer use tools from regular function tools
computer_use_tools, regular_tools = self._separate_computer_use_tools( computer_use_tools, regular_tools = self._separate_computer_use_tools(filtered_tools, model)
filtered_tools, model
)
# Process regular function tools using existing logic # Process regular function tools using existing logic
bedrock_tools = _bedrock_tools_pt(regular_tools) bedrock_tools = _bedrock_tools_pt(regular_tools)
@ -1365,9 +1263,7 @@ class AmazonConverseConfig(BaseConfig):
anthropic_beta_list.append(computer_use_header) anthropic_beta_list.append(computer_use_header)
# Transform computer use tools to proper Bedrock format # Transform computer use tools to proper Bedrock format
transformed_computer_tools = self._transform_computer_use_tools( transformed_computer_tools = self._transform_computer_use_tools(computer_use_tools)
computer_use_tools
)
additional_request_params["tools"] = transformed_computer_tools additional_request_params["tools"] = transformed_computer_tools
else: else:
# No computer use tools, process all tools as regular tools # No computer use tools, process all tools as regular tools
@ -1396,15 +1292,9 @@ class AmazonConverseConfig(BaseConfig):
""" """
Bedrock doesn't support tool calling without `tools=` param specified. Bedrock doesn't support tool calling without `tools=` param specified.
""" """
if ( if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages):
"tools" not in optional_params
and messages is not None
and has_tool_call_blocks(messages)
):
if litellm.modify_params: if litellm.modify_params:
optional_params["tools"] = add_dummy_tool( optional_params["tools"] = add_dummy_tool(custom_llm_provider="bedrock_converse")
custom_llm_provider="bedrock_converse"
)
else: else:
raise litellm.UnsupportedParamsError( raise litellm.UnsupportedParamsError(
message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.", message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.",
@ -1458,9 +1348,7 @@ class AmazonConverseConfig(BaseConfig):
bedrock_tool_config: Optional[ToolConfigBlock] = None bedrock_tool_config: Optional[ToolConfigBlock] = None
if len(bedrock_tools) > 0: if len(bedrock_tools) > 0:
tool_choice_values: ToolChoiceValuesBlock = inference_params.pop( tool_choice_values: ToolChoiceValuesBlock = inference_params.pop("tool_choice", None)
"tool_choice", None
)
bedrock_tool_config = ToolConfigBlock( bedrock_tool_config = ToolConfigBlock(
tools=bedrock_tools, tools=bedrock_tools,
) )
@ -1470,9 +1358,7 @@ class AmazonConverseConfig(BaseConfig):
data: CommonRequestObject = { data: CommonRequestObject = {
"additionalModelRequestFields": additional_request_params, "additionalModelRequestFields": additional_request_params,
"system": system_content_blocks, "system": system_content_blocks,
"inferenceConfig": self._transform_inference_params( "inferenceConfig": self._transform_inference_params(inference_params=inference_params),
inference_params=inference_params
),
} }
# Handle all config blocks # Handle all config blocks
@ -1502,14 +1388,10 @@ class AmazonConverseConfig(BaseConfig):
litellm_params: dict, litellm_params: dict,
headers: Optional[dict] = None, headers: Optional[dict] = None,
) -> RequestObject: ) -> RequestObject:
messages, system_content_blocks = self._transform_system_message( messages, system_content_blocks = self._transform_system_message(messages, model=model)
messages, model=model
)
# Convert last user message to guarded_text if guardrailConfig is present # Convert last user message to guarded_text if guardrailConfig is present
messages = self._convert_consecutive_user_messages_to_guarded_text( messages = self._convert_consecutive_user_messages_to_guarded_text(messages, optional_params)
messages, optional_params
)
## TRANSFORMATION ## ## TRANSFORMATION ##
_data: CommonRequestObject = self._transform_request_helper( _data: CommonRequestObject = self._transform_request_helper(
@ -1520,13 +1402,11 @@ class AmazonConverseConfig(BaseConfig):
headers=headers, headers=headers,
) )
bedrock_messages = ( bedrock_messages = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( messages=messages,
messages=messages, model=model,
model=model, llm_provider="bedrock_converse",
llm_provider="bedrock_converse", user_continue_message=litellm_params.pop("user_continue_message", None),
user_continue_message=litellm_params.pop("user_continue_message", None),
)
) )
data: RequestObject = {"messages": bedrock_messages, **_data} data: RequestObject = {"messages": bedrock_messages, **_data}
@ -1560,14 +1440,10 @@ class AmazonConverseConfig(BaseConfig):
litellm_params: dict, litellm_params: dict,
headers: Optional[dict] = None, headers: Optional[dict] = None,
) -> RequestObject: ) -> RequestObject:
messages, system_content_blocks = self._transform_system_message( messages, system_content_blocks = self._transform_system_message(messages, model=model)
messages, model=model
)
# Convert last user message to guarded_text if guardrailConfig is present # Convert last user message to guarded_text if guardrailConfig is present
messages = self._convert_consecutive_user_messages_to_guarded_text( messages = self._convert_consecutive_user_messages_to_guarded_text(messages, optional_params)
messages, optional_params
)
_data: CommonRequestObject = self._transform_request_helper( _data: CommonRequestObject = self._transform_request_helper(
model=model, model=model,
@ -1616,9 +1492,7 @@ class AmazonConverseConfig(BaseConfig):
encoding=encoding, encoding=encoding,
) )
def _transform_reasoning_content( def _transform_reasoning_content(self, reasoning_content_blocks: List[BedrockConverseReasoningContentBlock]) -> str:
self, reasoning_content_blocks: List[BedrockConverseReasoningContentBlock]
) -> str:
""" """
Extract the reasoning text from the reasoning content blocks Extract the reasoning text from the reasoning content blocks
@ -1634,9 +1508,7 @@ class AmazonConverseConfig(BaseConfig):
self, thinking_blocks: List[BedrockConverseReasoningContentBlock] self, thinking_blocks: List[BedrockConverseReasoningContentBlock]
) -> List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]: ) -> List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]:
"""Return a consistent format for thinking blocks between Anthropic and Bedrock.""" """Return a consistent format for thinking blocks between Anthropic and Bedrock."""
thinking_blocks_list: List[ thinking_blocks_list: List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] = []
Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
] = []
for block in thinking_blocks: for block in thinking_blocks:
if "reasoningText" in block: if "reasoningText" in block:
_thinking_block = ChatCompletionThinkingBlock(type="thinking") _thinking_block = ChatCompletionThinkingBlock(type="thinking")
@ -1672,21 +1544,11 @@ class AmazonConverseConfig(BaseConfig):
cache_creation_input_tokens = usage["cacheWriteInputTokens"] cache_creation_input_tokens = usage["cacheWriteInputTokens"]
input_tokens += cache_creation_input_tokens input_tokens += cache_creation_input_tokens
prompt_tokens_details = PromptTokensDetailsWrapper( prompt_tokens_details = PromptTokensDetailsWrapper(cached_tokens=cache_read_input_tokens)
cached_tokens=cache_read_input_tokens reasoning_tokens = token_counter(text=reasoning_content, count_response_tokens=True) if reasoning_content else 0
)
reasoning_tokens = (
token_counter(text=reasoning_content, count_response_tokens=True)
if reasoning_content
else 0
)
completion_tokens_details = CompletionTokensDetailsWrapper( completion_tokens_details = CompletionTokensDetailsWrapper(
reasoning_tokens=reasoning_tokens, reasoning_tokens=reasoning_tokens,
text_tokens=( text_tokens=(output_tokens - reasoning_tokens if reasoning_tokens > 0 else output_tokens),
output_tokens - reasoning_tokens
if reasoning_tokens > 0
else output_tokens
),
) )
openai_usage = Usage( openai_usage = Usage(
prompt_tokens=input_tokens, prompt_tokens=input_tokens,
@ -1701,9 +1563,7 @@ class AmazonConverseConfig(BaseConfig):
def get_tool_call_names( def get_tool_call_names(
self, self,
tools: Optional[ tools: Optional[Union[List[ToolBlock], List[OpenAIChatCompletionToolParam]]] = None,
Union[List[ToolBlock], List[OpenAIChatCompletionToolParam]]
] = None,
) -> List[str]: ) -> List[str]:
if tools is None: if tools is None:
return [] return []
@ -1742,13 +1602,8 @@ class AmazonConverseConfig(BaseConfig):
try: try:
tool_call_names = self.get_tool_call_names(tools) tool_call_names = self.get_tool_call_names(tools)
json_content = json.loads(message.content) json_content = json.loads(message.content)
if ( if json_content.get("type") == "function" and json_content.get("name") in tool_call_names:
json_content.get("type") == "function" tool_calls = [ChatCompletionMessageToolCall(function=Function(**json_content))]
and json_content.get("name") in tool_call_names
):
tool_calls = [
ChatCompletionMessageToolCall(function=Function(**json_content))
]
message.tool_calls = tool_calls message.tool_calls = tool_calls
message.content = None message.content = None
@ -1777,9 +1632,7 @@ class AmazonConverseConfig(BaseConfig):
""" """
content_str = "" content_str = ""
tools: List[ChatCompletionToolCallChunk] = [] tools: List[ChatCompletionToolCallChunk] = []
reasoningContentBlocks: Optional[ reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = None
List[BedrockConverseReasoningContentBlock]
] = None
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
for idx, content in enumerate(content_blocks): for idx, content in enumerate(content_blocks):
""" """
@ -1796,9 +1649,7 @@ class AmazonConverseConfig(BaseConfig):
if "toolUse" in content: if "toolUse" in content:
## check tool name was formatted by litellm ## check tool name was formatted by litellm
_response_tool_name = content["toolUse"]["name"] _response_tool_name = content["toolUse"]["name"]
response_tool_name = get_bedrock_tool_name( response_tool_name = get_bedrock_tool_name(response_tool_name=_response_tool_name)
response_tool_name=_response_tool_name
)
_function_chunk = ChatCompletionToolCallFunctionChunk( _function_chunk = ChatCompletionToolCallFunctionChunk(
name=response_tool_name, name=response_tool_name,
arguments=json.dumps(content["toolUse"]["input"]), arguments=json.dumps(content["toolUse"]["input"]),
@ -1849,11 +1700,7 @@ class AmazonConverseConfig(BaseConfig):
""" """
try: try:
response_data = json.loads(json_str) response_data = json.loads(json_str)
if ( if isinstance(response_data, dict) and "properties" in response_data and len(response_data) == 1:
isinstance(response_data, dict)
and "properties" in response_data
and len(response_data) == 1
):
response_data = response_data["properties"] response_data = response_data["properties"]
return json.dumps(response_data) return json.dumps(response_data)
except json.JSONDecodeError: except json.JSONDecodeError:
@ -1877,11 +1724,7 @@ class AmazonConverseConfig(BaseConfig):
if not json_mode or not tools: if not json_mode or not tools:
return tools if tools else None return tools if tools else None
json_tool_indices = [ json_tool_indices = [i for i, t in enumerate(tools) if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME]
i
for i, t in enumerate(tools)
if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME
]
if not json_tool_indices: if not json_tool_indices:
# No json_tool_call found, return tools unchanged # No json_tool_call found, return tools unchanged
@ -1889,14 +1732,10 @@ class AmazonConverseConfig(BaseConfig):
if len(json_tool_indices) == len(tools): if len(json_tool_indices) == len(tools):
# All tools are json_tool_call — convert first one to content # All tools are json_tool_call — convert first one to content
verbose_logger.debug( verbose_logger.debug("Processing JSON tool call response for response_format")
"Processing JSON tool call response for response_format"
)
json_mode_content_str: Optional[str] = tools[0]["function"].get("arguments") json_mode_content_str: Optional[str] = tools[0]["function"].get("arguments")
if json_mode_content_str is not None: if json_mode_content_str is not None:
json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties( json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties(json_mode_content_str)
json_mode_content_str
)
chat_completion_message["content"] = json_mode_content_str chat_completion_message["content"] = json_mode_content_str
return None return None
@ -1906,13 +1745,9 @@ class AmazonConverseConfig(BaseConfig):
first_idx = json_tool_indices[0] first_idx = json_tool_indices[0]
json_mode_args = tools[first_idx]["function"].get("arguments") json_mode_args = tools[first_idx]["function"].get("arguments")
if json_mode_args is not None: if json_mode_args is not None:
json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties( json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties(json_mode_args)
json_mode_args
)
existing = chat_completion_message.get("content") or "" existing = chat_completion_message.get("content") or ""
chat_completion_message["content"] = ( chat_completion_message["content"] = existing + json_mode_args if existing else json_mode_args
existing + json_mode_args if existing else json_mode_args
)
real_tools = [t for i, t in enumerate(tools) if i not in json_tool_indices] real_tools = [t for i, t in enumerate(tools) if i not in json_tool_indices]
return real_tools if real_tools else None return real_tools if real_tools else None
@ -1990,9 +1825,7 @@ class AmazonConverseConfig(BaseConfig):
chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"} chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
content_str = "" content_str = ""
tools: List[ChatCompletionToolCallChunk] = [] tools: List[ChatCompletionToolCallChunk] = []
reasoningContentBlocks: Optional[ reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = None
List[BedrockConverseReasoningContentBlock]
] = None
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
if message is not None: if message is not None:
@ -2011,17 +1844,11 @@ class AmazonConverseConfig(BaseConfig):
provider_specific_fields["citationsContent"] = citationsContentBlocks provider_specific_fields["citationsContent"] = citationsContentBlocks
if provider_specific_fields: if provider_specific_fields:
chat_completion_message[ chat_completion_message["provider_specific_fields"] = provider_specific_fields
"provider_specific_fields"
] = provider_specific_fields
if reasoningContentBlocks is not None: if reasoningContentBlocks is not None:
chat_completion_message[ chat_completion_message["reasoning_content"] = self._transform_reasoning_content(reasoningContentBlocks)
"reasoning_content" chat_completion_message["thinking_blocks"] = self._transform_thinking_blocks(reasoningContentBlocks)
] = self._transform_reasoning_content(reasoningContentBlocks)
chat_completion_message[
"thinking_blocks"
] = self._transform_thinking_blocks(reasoningContentBlocks)
chat_completion_message["content"] = content_str chat_completion_message["content"] = content_str
filtered_tools = self._filter_json_mode_tools( filtered_tools = self._filter_json_mode_tools(
json_mode=json_mode, json_mode=json_mode,

144
litellm/model_prices_and_context_window_backup.json
View File

@ -722,7 +722,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"anthropic.claude-haiku-4-5@20251001": { "anthropic.claude-haiku-4-5@20251001": {
"cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost": 1.25e-06,
@ -745,7 +746,8 @@
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346, "tool_use_system_prompt_tokens": 346,
"supports_native_streaming": true "supports_native_streaming": true,
"supports_native_structured_output": true
}, },
"anthropic.claude-3-5-sonnet-20240620-v1:0": { "anthropic.claude-3-5-sonnet-20240620-v1:0": {
"input_cost_per_token": 3e-06, "input_cost_per_token": 3e-06,
@ -967,7 +969,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 159 "tool_use_system_prompt_tokens": 159,
"supports_native_structured_output": true
}, },
"anthropic.claude-opus-4-6-v1": { "anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost": 6.25e-06,
@ -993,7 +996,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"global.anthropic.claude-opus-4-6-v1": { "global.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost": 6.25e-06,
@ -1019,7 +1023,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"us.anthropic.claude-opus-4-6-v1": { "us.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost": 6.875e-06,
@ -1045,7 +1050,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"eu.anthropic.claude-opus-4-6-v1": { "eu.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost": 6.875e-06,
@ -1071,7 +1077,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"au.anthropic.claude-opus-4-6-v1": { "au.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost": 6.875e-06,
@ -1097,7 +1104,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"anthropic.claude-sonnet-4-6": { "anthropic.claude-sonnet-4-6": {
"cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost": 3.75e-06,
@ -1123,7 +1131,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"global.anthropic.claude-sonnet-4-6": { "global.anthropic.claude-sonnet-4-6": {
"cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost": 3.75e-06,
@ -1149,7 +1158,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"us.anthropic.claude-sonnet-4-6": { "us.anthropic.claude-sonnet-4-6": {
"cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost": 4.125e-06,
@ -1175,7 +1185,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"eu.anthropic.claude-sonnet-4-6": { "eu.anthropic.claude-sonnet-4-6": {
"cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost": 4.125e-06,
@ -1201,7 +1212,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"au.anthropic.claude-sonnet-4-6": { "au.anthropic.claude-sonnet-4-6": {
"cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost": 4.125e-06,
@ -1227,7 +1239,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"anthropic.claude-sonnet-4-20250514-v1:0": { "anthropic.claude-sonnet-4-20250514-v1:0": {
"cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost": 3.75e-06,
@ -1287,7 +1300,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 159 "tool_use_system_prompt_tokens": 159,
"supports_native_structured_output": true
}, },
"anthropic.claude-v1": { "anthropic.claude-v1": {
"input_cost_per_token": 8e-06, "input_cost_per_token": 8e-06,
@ -1537,7 +1551,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"apac.anthropic.claude-3-sonnet-20240229-v1:0": { "apac.anthropic.claude-3-sonnet-20240229-v1:0": {
"input_cost_per_token": 3e-06, "input_cost_per_token": 3e-06,
@ -1625,7 +1640,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"azure/ada": { "azure/ada": {
"input_cost_per_token": 1e-07, "input_cost_per_token": 1e-07,
@ -11690,7 +11706,8 @@
"output_cost_per_token": 1.68e-06, "output_cost_per_token": 1.68e-06,
"supports_function_calling": true, "supports_function_calling": true,
"supports_reasoning": true, "supports_reasoning": true,
"supports_tool_choice": true "supports_tool_choice": true,
"supports_native_structured_output": true
}, },
"deepseek.v3.2": { "deepseek.v3.2": {
"input_cost_per_token": 6.2e-07, "input_cost_per_token": 6.2e-07,
@ -12135,7 +12152,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": {
"input_cost_per_token": 3e-06, "input_cost_per_token": 3e-06,
@ -12349,7 +12367,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"eu.meta.llama3-2-1b-instruct-v1:0": { "eu.meta.llama3-2-1b-instruct-v1:0": {
"input_cost_per_token": 1.3e-07, "input_cost_per_token": 1.3e-07,
@ -14579,18 +14598,6 @@
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"uses_embed_content": true "uses_embed_content": true
}, },
"vertex_ai/gemini-embedding-2-preview": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
"mode": "embedding",
"output_cost_per_token": 0,
"output_vector_size": 3072,
"source": "https://ai.google.dev/gemini-api/docs/embeddings#multimodal",
"supports_multimodal": true,
"uses_embed_content": true
},
"gemini/gemini-embedding-001": { "gemini/gemini-embedding-001": {
"input_cost_per_token": 1.5e-07, "input_cost_per_token": 1.5e-07,
"litellm_provider": "gemini", "litellm_provider": "gemini",
@ -16767,7 +16774,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"global.anthropic.claude-sonnet-4-20250514-v1:0": { "global.anthropic.claude-sonnet-4-20250514-v1:0": {
"cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost": 3.75e-06,
@ -16819,7 +16827,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"global.amazon.nova-2-lite-v1:0": { "global.amazon.nova-2-lite-v1:0": {
"cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost": 7.5e-08,
@ -20548,7 +20557,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "jp.anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost": 1.375e-06,
@ -20570,7 +20580,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"lambda_ai/deepseek-llama3.3-70b": { "lambda_ai/deepseek-llama3.3-70b": {
"input_cost_per_token": 2e-07, "input_cost_per_token": 2e-07,
@ -21265,7 +21276,8 @@
"max_tokens": 8192, "max_tokens": 8192,
"mode": "chat", "mode": "chat",
"output_cost_per_token": 1.2e-06, "output_cost_per_token": 1.2e-06,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"minimax.minimax-m2.1": { "minimax.minimax-m2.1": {
"input_cost_per_token": 3e-07, "input_cost_per_token": 3e-07,
@ -21421,7 +21433,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 2e-07, "output_cost_per_token": 2e-07,
"supports_function_calling": true, "supports_function_calling": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"mistral.ministral-3-3b-instruct": { "mistral.ministral-3-3b-instruct": {
"input_cost_per_token": 1e-07, "input_cost_per_token": 1e-07,
@ -21432,7 +21445,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 1e-07, "output_cost_per_token": 1e-07,
"supports_function_calling": true, "supports_function_calling": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"mistral.ministral-3-8b-instruct": { "mistral.ministral-3-8b-instruct": {
"input_cost_per_token": 1.5e-07, "input_cost_per_token": 1.5e-07,
@ -21443,7 +21457,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 1.5e-07, "output_cost_per_token": 1.5e-07,
"supports_function_calling": true, "supports_function_calling": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"mistral.mistral-7b-instruct-v0:2": { "mistral.mistral-7b-instruct-v0:2": {
"input_cost_per_token": 1.5e-07, "input_cost_per_token": 1.5e-07,
@ -21485,7 +21500,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 1.5e-06, "output_cost_per_token": 1.5e-06,
"supports_function_calling": true, "supports_function_calling": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"mistral.mistral-small-2402-v1:0": { "mistral.mistral-small-2402-v1:0": {
"input_cost_per_token": 1e-06, "input_cost_per_token": 1e-06,
@ -21516,7 +21532,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 4e-08, "output_cost_per_token": 4e-08,
"supports_audio_input": true, "supports_audio_input": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"mistral.voxtral-small-24b-2507": { "mistral.voxtral-small-24b-2507": {
"input_cost_per_token": 1e-07, "input_cost_per_token": 1e-07,
@ -21527,7 +21544,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 3e-07, "output_cost_per_token": 3e-07,
"supports_audio_input": true, "supports_audio_input": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"mistral/codestral-2405": { "mistral/codestral-2405": {
"input_cost_per_token": 1e-06, "input_cost_per_token": 1e-06,
@ -22214,7 +22232,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 2.5e-06, "output_cost_per_token": 2.5e-06,
"supports_reasoning": true, "supports_reasoning": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"moonshotai.kimi-k2.5": { "moonshotai.kimi-k2.5": {
"input_cost_per_token": 6e-07, "input_cost_per_token": 6e-07,
@ -23089,7 +23108,8 @@
"supports_function_calling": true, "supports_function_calling": true,
"supports_system_messages": true, "supports_system_messages": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/" "source": "https://aws.amazon.com/bedrock/pricing/",
"supports_native_structured_output": true
}, },
"o1": { "o1": {
"cache_read_input_token_cost": 7.5e-06, "cache_read_input_token_cost": 7.5e-06,
@ -26173,7 +26193,8 @@
"output_cost_per_token": 1.8e-06, "output_cost_per_token": 1.8e-06,
"supports_function_calling": true, "supports_function_calling": true,
"supports_reasoning": true, "supports_reasoning": true,
"supports_tool_choice": true "supports_tool_choice": true,
"supports_native_structured_output": true
}, },
"qwen.qwen3-235b-a22b-2507-v1:0": { "qwen.qwen3-235b-a22b-2507-v1:0": {
"input_cost_per_token": 2.2e-07, "input_cost_per_token": 2.2e-07,
@ -26185,7 +26206,8 @@
"output_cost_per_token": 8.8e-07, "output_cost_per_token": 8.8e-07,
"supports_function_calling": true, "supports_function_calling": true,
"supports_reasoning": true, "supports_reasoning": true,
"supports_tool_choice": true "supports_tool_choice": true,
"supports_native_structured_output": true
}, },
"qwen.qwen3-coder-30b-a3b-v1:0": { "qwen.qwen3-coder-30b-a3b-v1:0": {
"input_cost_per_token": 1.5e-07, "input_cost_per_token": 1.5e-07,
@ -26197,7 +26219,8 @@
"output_cost_per_token": 6e-07, "output_cost_per_token": 6e-07,
"supports_function_calling": true, "supports_function_calling": true,
"supports_reasoning": true, "supports_reasoning": true,
"supports_tool_choice": true "supports_tool_choice": true,
"supports_native_structured_output": true
}, },
"qwen.qwen3-32b-v1:0": { "qwen.qwen3-32b-v1:0": {
"input_cost_per_token": 1.5e-07, "input_cost_per_token": 1.5e-07,
@ -26209,7 +26232,8 @@
"output_cost_per_token": 6e-07, "output_cost_per_token": 6e-07,
"supports_function_calling": true, "supports_function_calling": true,
"supports_reasoning": true, "supports_reasoning": true,
"supports_tool_choice": true "supports_tool_choice": true,
"supports_native_structured_output": true
}, },
"qwen.qwen3-next-80b-a3b": { "qwen.qwen3-next-80b-a3b": {
"input_cost_per_token": 1.5e-07, "input_cost_per_token": 1.5e-07,
@ -26220,7 +26244,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 1.2e-06, "output_cost_per_token": 1.2e-06,
"supports_function_calling": true, "supports_function_calling": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"qwen.qwen3-vl-235b-a22b": { "qwen.qwen3-vl-235b-a22b": {
"input_cost_per_token": 5.3e-07, "input_cost_per_token": 5.3e-07,
@ -26232,7 +26257,8 @@
"output_cost_per_token": 2.66e-06, "output_cost_per_token": 2.66e-06,
"supports_function_calling": true, "supports_function_calling": true,
"supports_system_messages": true, "supports_system_messages": true,
"supports_vision": true "supports_vision": true,
"supports_native_structured_output": true
}, },
"qwen.qwen3-coder-next": { "qwen.qwen3-coder-next": {
"input_cost_per_token": 5e-07, "input_cost_per_token": 5e-07,
@ -28257,7 +28283,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"us.anthropic.claude-3-5-sonnet-20240620-v1:0": { "us.anthropic.claude-3-5-sonnet-20240620-v1:0": {
"input_cost_per_token": 3e-06, "input_cost_per_token": 3e-06,
@ -28415,7 +28442,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"au.anthropic.claude-haiku-4-5-20251001-v1:0": { "au.anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost": 1.375e-06,
@ -28436,7 +28464,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"us.anthropic.claude-opus-4-20250514-v1:0": { "us.anthropic.claude-opus-4-20250514-v1:0": {
"cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost": 1.875e-05,
@ -28488,7 +28517,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 159 "tool_use_system_prompt_tokens": 159,
"supports_native_structured_output": true
}, },
"global.anthropic.claude-opus-4-5-20251101-v1:0": { "global.anthropic.claude-opus-4-5-20251101-v1:0": {
"cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost": 6.25e-06,
@ -28514,7 +28544,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 159 "tool_use_system_prompt_tokens": 159,
"supports_native_structured_output": true
}, },
"eu.anthropic.claude-opus-4-5-20251101-v1:0": { "eu.anthropic.claude-opus-4-5-20251101-v1:0": {
"cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost": 6.25e-06,
@ -28540,7 +28571,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 159 "tool_use_system_prompt_tokens": 159,
"supports_native_structured_output": true
}, },
"us.anthropic.claude-sonnet-4-20250514-v1:0": { "us.anthropic.claude-sonnet-4-20250514-v1:0": {
"cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost": 3.75e-06,

1
litellm/types/utils.py
View File

@ -133,6 +133,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
supports_audio_output: Optional[bool] supports_audio_output: Optional[bool]
supports_pdf_input: Optional[bool] supports_pdf_input: Optional[bool]
supports_native_streaming: Optional[bool] supports_native_streaming: Optional[bool]
supports_native_structured_output: Optional[bool]
supports_parallel_function_calling: Optional[bool] supports_parallel_function_calling: Optional[bool]
supports_web_search: Optional[bool] supports_web_search: Optional[bool]
supports_reasoning: Optional[bool] supports_reasoning: Optional[bool]

16
litellm/utils.py
View File

@ -2735,6 +2735,19 @@ def supports_reasoning(model: str, custom_llm_provider: Optional[str] = None) ->
) )
def supports_native_structured_output(
model: str, custom_llm_provider: Optional[str] = None
) -> bool:
"""
Check if the given model supports native structured outputs and return a boolean value.
"""
return _supports_factory(
model=model,
custom_llm_provider=custom_llm_provider,
key="supports_native_structured_output",
)
def get_supported_regions( def get_supported_regions(
model: str, custom_llm_provider: Optional[str] = None model: str, custom_llm_provider: Optional[str] = None
) -> Optional[List[str]]: ) -> Optional[List[str]]:
@ -5845,6 +5858,9 @@ def _get_model_info_helper( # noqa: PLR0915
supports_native_streaming=_model_info.get( supports_native_streaming=_model_info.get(
"supports_native_streaming", None "supports_native_streaming", None
), ),
supports_native_structured_output=_model_info.get(
"supports_native_structured_output", None
),
supports_web_search=_model_info.get("supports_web_search", None), supports_web_search=_model_info.get("supports_web_search", None),
supports_url_context=_model_info.get("supports_url_context", None), supports_url_context=_model_info.get("supports_url_context", None),
supports_reasoning=_model_info.get("supports_reasoning", None), supports_reasoning=_model_info.get("supports_reasoning", None),

144
model_prices_and_context_window.json
View File

@ -722,7 +722,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"anthropic.claude-haiku-4-5@20251001": { "anthropic.claude-haiku-4-5@20251001": {
"cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost": 1.25e-06,
@ -745,7 +746,8 @@
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346, "tool_use_system_prompt_tokens": 346,
"supports_native_streaming": true "supports_native_streaming": true,
"supports_native_structured_output": true
}, },
"anthropic.claude-3-5-sonnet-20240620-v1:0": { "anthropic.claude-3-5-sonnet-20240620-v1:0": {
"input_cost_per_token": 3e-06, "input_cost_per_token": 3e-06,
@ -967,7 +969,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 159 "tool_use_system_prompt_tokens": 159,
"supports_native_structured_output": true
}, },
"anthropic.claude-opus-4-6-v1": { "anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost": 6.25e-06,
@ -993,7 +996,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"global.anthropic.claude-opus-4-6-v1": { "global.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost": 6.25e-06,
@ -1019,7 +1023,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"us.anthropic.claude-opus-4-6-v1": { "us.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost": 6.875e-06,
@ -1045,7 +1050,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"eu.anthropic.claude-opus-4-6-v1": { "eu.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost": 6.875e-06,
@ -1071,7 +1077,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"au.anthropic.claude-opus-4-6-v1": { "au.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost": 6.875e-06,
@ -1097,7 +1104,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"anthropic.claude-sonnet-4-6": { "anthropic.claude-sonnet-4-6": {
"cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost": 3.75e-06,
@ -1123,7 +1131,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"global.anthropic.claude-sonnet-4-6": { "global.anthropic.claude-sonnet-4-6": {
"cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost": 3.75e-06,
@ -1149,7 +1158,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"us.anthropic.claude-sonnet-4-6": { "us.anthropic.claude-sonnet-4-6": {
"cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost": 4.125e-06,
@ -1175,7 +1185,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"eu.anthropic.claude-sonnet-4-6": { "eu.anthropic.claude-sonnet-4-6": {
"cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost": 4.125e-06,
@ -1201,7 +1212,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"au.anthropic.claude-sonnet-4-6": { "au.anthropic.claude-sonnet-4-6": {
"cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost": 4.125e-06,
@ -1227,7 +1239,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"anthropic.claude-sonnet-4-20250514-v1:0": { "anthropic.claude-sonnet-4-20250514-v1:0": {
"cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost": 3.75e-06,
@ -1287,7 +1300,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 159 "tool_use_system_prompt_tokens": 159,
"supports_native_structured_output": true
}, },
"anthropic.claude-v1": { "anthropic.claude-v1": {
"input_cost_per_token": 8e-06, "input_cost_per_token": 8e-06,
@ -1537,7 +1551,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"apac.anthropic.claude-3-sonnet-20240229-v1:0": { "apac.anthropic.claude-3-sonnet-20240229-v1:0": {
"input_cost_per_token": 3e-06, "input_cost_per_token": 3e-06,
@ -1625,7 +1640,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"azure/ada": { "azure/ada": {
"input_cost_per_token": 1e-07, "input_cost_per_token": 1e-07,
@ -11690,7 +11706,8 @@
"output_cost_per_token": 1.68e-06, "output_cost_per_token": 1.68e-06,
"supports_function_calling": true, "supports_function_calling": true,
"supports_reasoning": true, "supports_reasoning": true,
"supports_tool_choice": true "supports_tool_choice": true,
"supports_native_structured_output": true
}, },
"deepseek.v3.2": { "deepseek.v3.2": {
"input_cost_per_token": 6.2e-07, "input_cost_per_token": 6.2e-07,
@ -12135,7 +12152,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": {
"input_cost_per_token": 3e-06, "input_cost_per_token": 3e-06,
@ -12349,7 +12367,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"eu.meta.llama3-2-1b-instruct-v1:0": { "eu.meta.llama3-2-1b-instruct-v1:0": {
"input_cost_per_token": 1.3e-07, "input_cost_per_token": 1.3e-07,
@ -14579,18 +14598,6 @@
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"uses_embed_content": true "uses_embed_content": true
}, },
"vertex_ai/gemini-embedding-2-preview": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
"mode": "embedding",
"output_cost_per_token": 0,
"output_vector_size": 3072,
"source": "https://ai.google.dev/gemini-api/docs/embeddings#multimodal",
"supports_multimodal": true,
"uses_embed_content": true
},
"gemini/gemini-embedding-001": { "gemini/gemini-embedding-001": {
"input_cost_per_token": 1.5e-07, "input_cost_per_token": 1.5e-07,
"litellm_provider": "gemini", "litellm_provider": "gemini",
@ -16767,7 +16774,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"global.anthropic.claude-sonnet-4-20250514-v1:0": { "global.anthropic.claude-sonnet-4-20250514-v1:0": {
"cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost": 3.75e-06,
@ -16819,7 +16827,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"global.amazon.nova-2-lite-v1:0": { "global.amazon.nova-2-lite-v1:0": {
"cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost": 7.5e-08,
@ -20548,7 +20557,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "jp.anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost": 1.375e-06,
@ -20570,7 +20580,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"lambda_ai/deepseek-llama3.3-70b": { "lambda_ai/deepseek-llama3.3-70b": {
"input_cost_per_token": 2e-07, "input_cost_per_token": 2e-07,
@ -21265,7 +21276,8 @@
"max_tokens": 8192, "max_tokens": 8192,
"mode": "chat", "mode": "chat",
"output_cost_per_token": 1.2e-06, "output_cost_per_token": 1.2e-06,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"minimax.minimax-m2.1": { "minimax.minimax-m2.1": {
"input_cost_per_token": 3e-07, "input_cost_per_token": 3e-07,
@ -21421,7 +21433,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 2e-07, "output_cost_per_token": 2e-07,
"supports_function_calling": true, "supports_function_calling": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"mistral.ministral-3-3b-instruct": { "mistral.ministral-3-3b-instruct": {
"input_cost_per_token": 1e-07, "input_cost_per_token": 1e-07,
@ -21432,7 +21445,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 1e-07, "output_cost_per_token": 1e-07,
"supports_function_calling": true, "supports_function_calling": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"mistral.ministral-3-8b-instruct": { "mistral.ministral-3-8b-instruct": {
"input_cost_per_token": 1.5e-07, "input_cost_per_token": 1.5e-07,
@ -21443,7 +21457,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 1.5e-07, "output_cost_per_token": 1.5e-07,
"supports_function_calling": true, "supports_function_calling": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"mistral.mistral-7b-instruct-v0:2": { "mistral.mistral-7b-instruct-v0:2": {
"input_cost_per_token": 1.5e-07, "input_cost_per_token": 1.5e-07,
@ -21485,7 +21500,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 1.5e-06, "output_cost_per_token": 1.5e-06,
"supports_function_calling": true, "supports_function_calling": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"mistral.mistral-small-2402-v1:0": { "mistral.mistral-small-2402-v1:0": {
"input_cost_per_token": 1e-06, "input_cost_per_token": 1e-06,
@ -21516,7 +21532,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 4e-08, "output_cost_per_token": 4e-08,
"supports_audio_input": true, "supports_audio_input": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"mistral.voxtral-small-24b-2507": { "mistral.voxtral-small-24b-2507": {
"input_cost_per_token": 1e-07, "input_cost_per_token": 1e-07,
@ -21527,7 +21544,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 3e-07, "output_cost_per_token": 3e-07,
"supports_audio_input": true, "supports_audio_input": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"mistral/codestral-2405": { "mistral/codestral-2405": {
"input_cost_per_token": 1e-06, "input_cost_per_token": 1e-06,
@ -22214,7 +22232,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 2.5e-06, "output_cost_per_token": 2.5e-06,
"supports_reasoning": true, "supports_reasoning": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"moonshotai.kimi-k2.5": { "moonshotai.kimi-k2.5": {
"input_cost_per_token": 6e-07, "input_cost_per_token": 6e-07,
@ -23089,7 +23108,8 @@
"supports_function_calling": true, "supports_function_calling": true,
"supports_system_messages": true, "supports_system_messages": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/" "source": "https://aws.amazon.com/bedrock/pricing/",
"supports_native_structured_output": true
}, },
"o1": { "o1": {
"cache_read_input_token_cost": 7.5e-06, "cache_read_input_token_cost": 7.5e-06,
@ -26173,7 +26193,8 @@
"output_cost_per_token": 1.8e-06, "output_cost_per_token": 1.8e-06,
"supports_function_calling": true, "supports_function_calling": true,
"supports_reasoning": true, "supports_reasoning": true,
"supports_tool_choice": true "supports_tool_choice": true,
"supports_native_structured_output": true
}, },
"qwen.qwen3-235b-a22b-2507-v1:0": { "qwen.qwen3-235b-a22b-2507-v1:0": {
"input_cost_per_token": 2.2e-07, "input_cost_per_token": 2.2e-07,
@ -26185,7 +26206,8 @@
"output_cost_per_token": 8.8e-07, "output_cost_per_token": 8.8e-07,
"supports_function_calling": true, "supports_function_calling": true,
"supports_reasoning": true, "supports_reasoning": true,
"supports_tool_choice": true "supports_tool_choice": true,
"supports_native_structured_output": true
}, },
"qwen.qwen3-coder-30b-a3b-v1:0": { "qwen.qwen3-coder-30b-a3b-v1:0": {
"input_cost_per_token": 1.5e-07, "input_cost_per_token": 1.5e-07,
@ -26197,7 +26219,8 @@
"output_cost_per_token": 6e-07, "output_cost_per_token": 6e-07,
"supports_function_calling": true, "supports_function_calling": true,
"supports_reasoning": true, "supports_reasoning": true,
"supports_tool_choice": true "supports_tool_choice": true,
"supports_native_structured_output": true
}, },
"qwen.qwen3-32b-v1:0": { "qwen.qwen3-32b-v1:0": {
"input_cost_per_token": 1.5e-07, "input_cost_per_token": 1.5e-07,
@ -26209,7 +26232,8 @@
"output_cost_per_token": 6e-07, "output_cost_per_token": 6e-07,
"supports_function_calling": true, "supports_function_calling": true,
"supports_reasoning": true, "supports_reasoning": true,
"supports_tool_choice": true "supports_tool_choice": true,
"supports_native_structured_output": true
}, },
"qwen.qwen3-next-80b-a3b": { "qwen.qwen3-next-80b-a3b": {
"input_cost_per_token": 1.5e-07, "input_cost_per_token": 1.5e-07,
@ -26220,7 +26244,8 @@
"mode": "chat", "mode": "chat",
"output_cost_per_token": 1.2e-06, "output_cost_per_token": 1.2e-06,
"supports_function_calling": true, "supports_function_calling": true,
"supports_system_messages": true "supports_system_messages": true,
"supports_native_structured_output": true
}, },
"qwen.qwen3-vl-235b-a22b": { "qwen.qwen3-vl-235b-a22b": {
"input_cost_per_token": 5.3e-07, "input_cost_per_token": 5.3e-07,
@ -26232,7 +26257,8 @@
"output_cost_per_token": 2.66e-06, "output_cost_per_token": 2.66e-06,
"supports_function_calling": true, "supports_function_calling": true,
"supports_system_messages": true, "supports_system_messages": true,
"supports_vision": true "supports_vision": true,
"supports_native_structured_output": true
}, },
"qwen.qwen3-coder-next": { "qwen.qwen3-coder-next": {
"input_cost_per_token": 5e-07, "input_cost_per_token": 5e-07,
@ -28257,7 +28283,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"us.anthropic.claude-3-5-sonnet-20240620-v1:0": { "us.anthropic.claude-3-5-sonnet-20240620-v1:0": {
"input_cost_per_token": 3e-06, "input_cost_per_token": 3e-06,
@ -28415,7 +28442,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"au.anthropic.claude-haiku-4-5-20251001-v1:0": { "au.anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost": 1.375e-06,
@ -28436,7 +28464,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 346 "tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
}, },
"us.anthropic.claude-opus-4-20250514-v1:0": { "us.anthropic.claude-opus-4-20250514-v1:0": {
"cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost": 1.875e-05,
@ -28488,7 +28517,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 159 "tool_use_system_prompt_tokens": 159,
"supports_native_structured_output": true
}, },
"global.anthropic.claude-opus-4-5-20251101-v1:0": { "global.anthropic.claude-opus-4-5-20251101-v1:0": {
"cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost": 6.25e-06,
@ -28514,7 +28544,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 159 "tool_use_system_prompt_tokens": 159,
"supports_native_structured_output": true
}, },
"eu.anthropic.claude-opus-4-5-20251101-v1:0": { "eu.anthropic.claude-opus-4-5-20251101-v1:0": {
"cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost": 6.25e-06,
@ -28540,7 +28571,8 @@
"supports_response_schema": true, "supports_response_schema": true,
"supports_tool_choice": true, "supports_tool_choice": true,
"supports_vision": true, "supports_vision": true,
"tool_use_system_prompt_tokens": 159 "tool_use_system_prompt_tokens": 159,
"supports_native_structured_output": true
}, },
"us.anthropic.claude-sonnet-4-20250514-v1:0": { "us.anthropic.claude-sonnet-4-20250514-v1:0": {
"cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost": 3.75e-06,

1214
tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
View File

File diff suppressed because it is too large Load Diff

1
tests/test_litellm/test_utils.py
View File

@ -831,6 +831,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
}, },
}, },
"supports_native_streaming": {"type": "boolean"}, "supports_native_streaming": {"type": "boolean"},
"supports_native_structured_output": {"type": "boolean"},
"tiered_pricing": { "tiered_pricing": {
"type": "array", "type": "array",
"items": { "items": {