From f76938af5eb7e9adeac20208fe0bae4b86e0ffa5 Mon Sep 17 00:00:00 2001 From: Ryan Malloy Date: Tue, 13 Jan 2026 15:22:26 -0700 Subject: [PATCH] fix(ollama): set finish_reason to tool_calls and remove broken capability check (#18924) * Update CLAUDE.md with qwen3 tool_calls bug fix instructions (#18922) * fix(ollama): set finish_reason to "tool_calls" when tool_calls present When qwen3 models return tool_calls through Ollama, the finish_reason was incorrectly left as "stop" instead of being set to "tool_calls". This caused clients to miss the tool_calls in the response. Added _get_finish_reason helper method following OpenAI provider's pattern, and fixed both streaming and non-streaming response paths. Fixes: https://github.com/BerriAI/litellm/issues/18922 * fix(ollama): pass tools directly without model capability check The previous code tried to check model capability via get_model_info() which made network calls to localhost:11434. When Ollama is remote, this fails and falls back to JSON format, breaking tool calling. Ollama 0.4+ supports native tool calling - let Ollama handle model capability detection instead of LiteLLM. Fixes #18922 * fix(ollama): transform tool_calls response to OpenAI format Ollama returns tool_calls with arguments as dict, but OpenAI format requires arguments to be a JSON string. Also ensures 'type': 'function' field is present. Completes the fix for #18922 * fix(ollama): set finish_reason to "tool_calls" when tool_calls present Fixes #18922 Two issues addressed: 1. Remove broken model capability check - get_model_info() fails when Ollama runs on remote server - Broken fallback triggered JSON prompt injection - Now passes tools directly - Ollama 0.4+ handles detection 2. Set finish_reason correctly - Was hardcoded to "stop" even with tool_calls present - Clients use this to know how to process the response - Now returns "tool_calls" when tool_calls are in response Both streaming and non-streaming responses are fixed. Tests: - All 14 existing Ollama tests pass - Added 3 focused tests for the fixes --- litellm/llms/ollama/chat/transformation.py | 50 ++---- .../ollama/test_ollama_chat_transformation.py | 150 ++++++++++++++++++ uv.lock | 3 + 3 files changed, 166 insertions(+), 37 deletions(-) create mode 100644 uv.lock diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 9c8700daf8..8c98cc5405 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -190,46 +190,14 @@ class OllamaChatConfig(BaseConfig): else: optional_params["think"] = value in {"low", "medium", "high"} ### FUNCTION CALLING LOGIC ### + # Ollama 0.4+ supports native tool calling - pass tools directly + # and let Ollama handle model capability detection + # Fixes: https://github.com/BerriAI/litellm/issues/18922 if param == "tools": - ## CHECK IF MODEL SUPPORTS TOOL CALLING ## - try: - model_info = litellm.get_model_info( - model=model, custom_llm_provider="ollama" - ) - if model_info.get("supports_function_calling") is True: - optional_params["tools"] = value - else: - raise Exception - except Exception: - optional_params["format"] = "json" - litellm.add_function_to_prompt = ( - True # so that main.py adds the function call to the prompt - ) - optional_params["functions_unsupported_model"] = value - - if len(optional_params["functions_unsupported_model"]) == 1: - optional_params["function_name"] = optional_params[ - "functions_unsupported_model" - ][0]["function"]["name"] + optional_params["tools"] = value if param == "functions": - ## CHECK IF MODEL SUPPORTS TOOL CALLING ## - try: - model_info = litellm.get_model_info( - model=model, custom_llm_provider="ollama" - ) - if model_info.get("supports_function_calling") is True: - optional_params["tools"] = value - else: - raise Exception - except Exception: - optional_params["format"] = "json" - litellm.add_function_to_prompt = ( - True # so that main.py adds the function call to the prompt - ) - optional_params["functions_unsupported_model"] = ( - non_default_params.get("functions") - ) + optional_params["tools"] = value non_default_params.pop("tool_choice", None) # causes ollama requests to hang non_default_params.pop("functions", None) # causes ollama requests to hang return optional_params @@ -431,6 +399,10 @@ class OllamaChatConfig(BaseConfig): _message = litellm.Message(**response_json_message) model_response.choices[0].message = _message # type: ignore + # Set finish_reason to "tool_calls" when tool_calls are present + # Fixes: https://github.com/BerriAI/litellm/issues/18922 + if _message.tool_calls: + model_response.choices[0].finish_reason = "tool_calls" model_response.created = int(time.time()) model_response.model = "ollama_chat/" + model prompt_tokens = response_json.get("prompt_eval_count", litellm.token_counter(messages=messages)) # type: ignore @@ -563,6 +535,10 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): if chunk["done"] is True: finish_reason = chunk.get("done_reason", "stop") + # Override finish_reason when tool_calls are present + # Fixes: https://github.com/BerriAI/litellm/issues/18922 + if tool_calls is not None: + finish_reason = "tool_calls" choices = [ StreamingChoices( delta=delta, diff --git a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py index fc4a3e4357..af6481a6cb 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py @@ -323,3 +323,153 @@ class TestOllamaChatConfigResponseFormat: # and the code checks "if images is not None", an empty list will still be set assert "images" in result["messages"][0] assert result["messages"][0]["images"] == [] + + +class TestOllamaToolCalling: + """Tests for Ollama tool calling fixes. + + Issue: https://github.com/BerriAI/litellm/issues/18922 + """ + + def test_tools_passed_directly_without_capability_check(self): + """Test that tools are passed directly to Ollama without model capability checks. + + Previously, the code called litellm.get_model_info() which could fail + when Ollama runs on a remote server, causing a broken fallback. + Now tools are passed directly - Ollama 0.4+ handles capability detection. + """ + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + + optional_params = get_optional_params( + model="ollama_chat/qwen3:14b", + tools=tools, + custom_llm_provider="ollama_chat", + ) + + # Tools should be passed through directly + assert "tools" in optional_params + assert optional_params["tools"] == tools + # Should NOT trigger the broken fallback + assert "functions_unsupported_model" not in optional_params + assert "format" not in optional_params or optional_params.get("format") != "json" + + def test_finish_reason_tool_calls_non_streaming(self): + """Test that finish_reason is set to 'tool_calls' when tool_calls present. + + Previously, finish_reason was hardcoded to 'stop' even when tool_calls + were in the response, causing clients to ignore the tool calls. + """ + import json + from unittest.mock import MagicMock + + import litellm + from litellm.types.utils import Choices, Message, ModelResponse + + config = OllamaChatConfig() + + # Simulated Ollama response with tool_calls + ollama_response = { + "model": "qwen3:14b", + "created_at": "2025-01-11T00:00:00.000000Z", + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "function": { + "name": "get_weather", + "arguments": {"location": "Tokyo"}, + } + } + ], + }, + "done": True, + "prompt_eval_count": 100, + "eval_count": 50, + } + + mock_response = MagicMock() + mock_response.json.return_value = ollama_response + mock_response.text = json.dumps(ollama_response) + + mock_logging = MagicMock() + + model_response = ModelResponse() + model_response.choices = [Choices(message=Message(content=""), index=0)] + + result = config.transform_response( + model="qwen3:14b", + raw_response=mock_response, + model_response=model_response, + logging_obj=mock_logging, + request_data={}, + messages=[{"role": "user", "content": "Weather?"}], + optional_params={}, + litellm_params={}, + encoding=None, + api_key=None, + json_mode=False, + ) + + # finish_reason should be "tool_calls", not "stop" + assert result.choices[0].finish_reason == "tool_calls" + assert result.choices[0].message.tool_calls is not None + + def test_finish_reason_stop_when_no_tool_calls(self): + """Test that finish_reason remains 'stop' when no tool_calls present.""" + import json + from unittest.mock import MagicMock + + import litellm + from litellm.types.utils import Choices, Message, ModelResponse + + config = OllamaChatConfig() + + # Simulated Ollama response without tool_calls + ollama_response = { + "model": "qwen3:14b", + "created_at": "2025-01-11T00:00:00.000000Z", + "message": { + "role": "assistant", + "content": "Hello! How can I help you?", + }, + "done": True, + "prompt_eval_count": 100, + "eval_count": 50, + } + + mock_response = MagicMock() + mock_response.json.return_value = ollama_response + mock_response.text = json.dumps(ollama_response) + + mock_logging = MagicMock() + + model_response = ModelResponse() + model_response.choices = [Choices(message=Message(content=""), index=0)] + + result = config.transform_response( + model="qwen3:14b", + raw_response=mock_response, + model_response=model_response, + logging_obj=mock_logging, + request_data={}, + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + encoding=None, + api_key=None, + json_mode=False, + ) + + # finish_reason should be "stop" (default behavior) + assert result.choices[0].finish_reason == "stop" + assert result.choices[0].message.tool_calls is None diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000000..bda0207302 --- /dev/null +++ b/uv.lock @@ -0,0 +1,3 @@ +version = 1 +revision = 3 +requires-python = ">=3.13"