fix(ollama): set finish_reason to tool_calls and remove broken capability check (#18924)
* Update CLAUDE.md with qwen3 tool_calls bug fix instructions (#18922) * fix(ollama): set finish_reason to "tool_calls" when tool_calls present When qwen3 models return tool_calls through Ollama, the finish_reason was incorrectly left as "stop" instead of being set to "tool_calls". This caused clients to miss the tool_calls in the response. Added _get_finish_reason helper method following OpenAI provider's pattern, and fixed both streaming and non-streaming response paths. Fixes: https://github.com/BerriAI/litellm/issues/18922 * fix(ollama): pass tools directly without model capability check The previous code tried to check model capability via get_model_info() which made network calls to localhost:11434. When Ollama is remote, this fails and falls back to JSON format, breaking tool calling. Ollama 0.4+ supports native tool calling - let Ollama handle model capability detection instead of LiteLLM. Fixes #18922 * fix(ollama): transform tool_calls response to OpenAI format Ollama returns tool_calls with arguments as dict, but OpenAI format requires arguments to be a JSON string. Also ensures 'type': 'function' field is present. Completes the fix for #18922 * fix(ollama): set finish_reason to "tool_calls" when tool_calls present Fixes #18922 Two issues addressed: 1. Remove broken model capability check - get_model_info() fails when Ollama runs on remote server - Broken fallback triggered JSON prompt injection - Now passes tools directly - Ollama 0.4+ handles detection 2. Set finish_reason correctly - Was hardcoded to "stop" even with tool_calls present - Clients use this to know how to process the response - Now returns "tool_calls" when tool_calls are in response Both streaming and non-streaming responses are fixed. Tests: - All 14 existing Ollama tests pass - Added 3 focused tests for the fixes
This commit is contained in:
Ryan Malloy
GitHub
parent
f8836cb2a7
commit
f76938af5e
@ -190,46 +190,14 @@ class OllamaChatConfig(BaseConfig):
|
||||
else:
|
||||
optional_params["think"] = value in {"low", "medium", "high"}
|
||||
### FUNCTION CALLING LOGIC ###
|
||||
# Ollama 0.4+ supports native tool calling - pass tools directly
|
||||
# and let Ollama handle model capability detection
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/18922
|
||||
if param == "tools":
|
||||
## CHECK IF MODEL SUPPORTS TOOL CALLING ##
|
||||
try:
|
||||
model_info = litellm.get_model_info(
|
||||
model=model, custom_llm_provider="ollama"
|
||||
)
|
||||
if model_info.get("supports_function_calling") is True:
|
||||
optional_params["tools"] = value
|
||||
else:
|
||||
raise Exception
|
||||
except Exception:
|
||||
optional_params["format"] = "json"
|
||||
litellm.add_function_to_prompt = (
|
||||
True # so that main.py adds the function call to the prompt
|
||||
)
|
||||
optional_params["functions_unsupported_model"] = value
|
||||
|
||||
if len(optional_params["functions_unsupported_model"]) == 1:
|
||||
optional_params["function_name"] = optional_params[
|
||||
"functions_unsupported_model"
|
||||
][0]["function"]["name"]
|
||||
optional_params["tools"] = value
|
||||
|
||||
if param == "functions":
|
||||
## CHECK IF MODEL SUPPORTS TOOL CALLING ##
|
||||
try:
|
||||
model_info = litellm.get_model_info(
|
||||
model=model, custom_llm_provider="ollama"
|
||||
)
|
||||
if model_info.get("supports_function_calling") is True:
|
||||
optional_params["tools"] = value
|
||||
else:
|
||||
raise Exception
|
||||
except Exception:
|
||||
optional_params["format"] = "json"
|
||||
litellm.add_function_to_prompt = (
|
||||
True # so that main.py adds the function call to the prompt
|
||||
)
|
||||
optional_params["functions_unsupported_model"] = (
|
||||
non_default_params.get("functions")
|
||||
)
|
||||
optional_params["tools"] = value
|
||||
non_default_params.pop("tool_choice", None) # causes ollama requests to hang
|
||||
non_default_params.pop("functions", None) # causes ollama requests to hang
|
||||
return optional_params
|
||||
@ -431,6 +399,10 @@ class OllamaChatConfig(BaseConfig):
|
||||
|
||||
_message = litellm.Message(**response_json_message)
|
||||
model_response.choices[0].message = _message # type: ignore
|
||||
# Set finish_reason to "tool_calls" when tool_calls are present
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/18922
|
||||
if _message.tool_calls:
|
||||
model_response.choices[0].finish_reason = "tool_calls"
|
||||
model_response.created = int(time.time())
|
||||
model_response.model = "ollama_chat/" + model
|
||||
prompt_tokens = response_json.get("prompt_eval_count", litellm.token_counter(messages=messages)) # type: ignore
|
||||
@ -563,6 +535,10 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
|
||||
|
||||
if chunk["done"] is True:
|
||||
finish_reason = chunk.get("done_reason", "stop")
|
||||
# Override finish_reason when tool_calls are present
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/18922
|
||||
if tool_calls is not None:
|
||||
finish_reason = "tool_calls"
|
||||
choices = [
|
||||
StreamingChoices(
|
||||
delta=delta,
|
||||
|
||||
@ -323,3 +323,153 @@ class TestOllamaChatConfigResponseFormat:
|
||||
# and the code checks "if images is not None", an empty list will still be set
|
||||
assert "images" in result["messages"][0]
|
||||
assert result["messages"][0]["images"] == []
|
||||
|
||||
|
||||
class TestOllamaToolCalling:
|
||||
"""Tests for Ollama tool calling fixes.
|
||||
|
||||
Issue: https://github.com/BerriAI/litellm/issues/18922
|
||||
"""
|
||||
|
||||
def test_tools_passed_directly_without_capability_check(self):
|
||||
"""Test that tools are passed directly to Ollama without model capability checks.
|
||||
|
||||
Previously, the code called litellm.get_model_info() which could fail
|
||||
when Ollama runs on a remote server, causing a broken fallback.
|
||||
Now tools are passed directly - Ollama 0.4+ handles capability detection.
|
||||
"""
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
optional_params = get_optional_params(
|
||||
model="ollama_chat/qwen3:14b",
|
||||
tools=tools,
|
||||
custom_llm_provider="ollama_chat",
|
||||
)
|
||||
|
||||
# Tools should be passed through directly
|
||||
assert "tools" in optional_params
|
||||
assert optional_params["tools"] == tools
|
||||
# Should NOT trigger the broken fallback
|
||||
assert "functions_unsupported_model" not in optional_params
|
||||
assert "format" not in optional_params or optional_params.get("format") != "json"
|
||||
|
||||
def test_finish_reason_tool_calls_non_streaming(self):
|
||||
"""Test that finish_reason is set to 'tool_calls' when tool_calls present.
|
||||
|
||||
Previously, finish_reason was hardcoded to 'stop' even when tool_calls
|
||||
were in the response, causing clients to ignore the tool calls.
|
||||
"""
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import litellm
|
||||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
|
||||
config = OllamaChatConfig()
|
||||
|
||||
# Simulated Ollama response with tool_calls
|
||||
ollama_response = {
|
||||
"model": "qwen3:14b",
|
||||
"created_at": "2025-01-11T00:00:00.000000Z",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": {"location": "Tokyo"},
|
||||
}
|
||||
}
|
||||
],
|
||||
},
|
||||
"done": True,
|
||||
"prompt_eval_count": 100,
|
||||
"eval_count": 50,
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = ollama_response
|
||||
mock_response.text = json.dumps(ollama_response)
|
||||
|
||||
mock_logging = MagicMock()
|
||||
|
||||
model_response = ModelResponse()
|
||||
model_response.choices = [Choices(message=Message(content=""), index=0)]
|
||||
|
||||
result = config.transform_response(
|
||||
model="qwen3:14b",
|
||||
raw_response=mock_response,
|
||||
model_response=model_response,
|
||||
logging_obj=mock_logging,
|
||||
request_data={},
|
||||
messages=[{"role": "user", "content": "Weather?"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
api_key=None,
|
||||
json_mode=False,
|
||||
)
|
||||
|
||||
# finish_reason should be "tool_calls", not "stop"
|
||||
assert result.choices[0].finish_reason == "tool_calls"
|
||||
assert result.choices[0].message.tool_calls is not None
|
||||
|
||||
def test_finish_reason_stop_when_no_tool_calls(self):
|
||||
"""Test that finish_reason remains 'stop' when no tool_calls present."""
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import litellm
|
||||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
|
||||
config = OllamaChatConfig()
|
||||
|
||||
# Simulated Ollama response without tool_calls
|
||||
ollama_response = {
|
||||
"model": "qwen3:14b",
|
||||
"created_at": "2025-01-11T00:00:00.000000Z",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Hello! How can I help you?",
|
||||
},
|
||||
"done": True,
|
||||
"prompt_eval_count": 100,
|
||||
"eval_count": 50,
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = ollama_response
|
||||
mock_response.text = json.dumps(ollama_response)
|
||||
|
||||
mock_logging = MagicMock()
|
||||
|
||||
model_response = ModelResponse()
|
||||
model_response.choices = [Choices(message=Message(content=""), index=0)]
|
||||
|
||||
result = config.transform_response(
|
||||
model="qwen3:14b",
|
||||
raw_response=mock_response,
|
||||
model_response=model_response,
|
||||
logging_obj=mock_logging,
|
||||
request_data={},
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
api_key=None,
|
||||
json_mode=False,
|
||||
)
|
||||
|
||||
# finish_reason should be "stop" (default behavior)
|
||||
assert result.choices[0].finish_reason == "stop"
|
||||
assert result.choices[0].message.tool_calls is None
|
||||
|
||||
Loading…
Reference in New Issue
Block a user