From 8e22cf5d6561c9bbe510ecbfd3403a72ae69a05a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 15 Sep 2025 18:49:54 -0700 Subject: [PATCH] [Fix] /responses API - add cancel endpoint + allow non-admins to use this as an llm api endpoint (#14594) * fix: ensure /responses/cancel works for non admins * test: cancel endpoint * fix responses API cancel endpoint * test fix * TestGoogleAIStudioResponsesAPITest --- litellm/proxy/_types.py | 2 + .../base_responses_api.py | 70 ++++++++------- .../test_anthropic_responses_api.py | 13 ++- .../test_google_ai_studio_responses_api.py | 13 ++- .../test_e2e_openai_responses_api.py | 90 +++++++++++-------- .../scim/test_scim_v2_endpoints.py | 10 ++- 6 files changed, 116 insertions(+), 82 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 4bd539ede4..2ef67c507b 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -312,6 +312,8 @@ class LiteLLMRoutes(enum.Enum): "/v1/responses/{response_id}", "/responses/{response_id}/input_items", "/v1/responses/{response_id}/input_items", + "/responses/{response_id}/cancel", + "/v1/responses/{response_id}/cancel", # vector stores "/vector_stores", "/v1/vector_stores", diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index 5ed4fbbb7b..8436f130e1 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -595,41 +595,47 @@ class BaseResponsesAPITest(ABC): @pytest.mark.flaky(retries=3, delay=2) @pytest.mark.asyncio async def test_basic_openai_responses_cancel_endpoint(self, sync_mode): - litellm._turn_on_debug() - litellm.set_verbose = True - base_completion_call_args = self.get_base_completion_call_args() - if sync_mode: - response = litellm.responses( - input="Basic ping", max_output_tokens=20, background=True, **base_completion_call_args - ) - - # cancel the response - if isinstance(response, ResponsesAPIResponse): - cancel_result = litellm.cancel_responses( - response_id=response.id, **base_completion_call_args + try: + litellm._turn_on_debug() + litellm.set_verbose = True + base_completion_call_args = self.get_base_completion_call_args() + if sync_mode: + response = litellm.responses( + input="Basic ping", max_output_tokens=20, background=True, **base_completion_call_args ) - assert cancel_result is not None - assert hasattr(cancel_result, "id") - # The actual response structure depends on the provider implementation - assert isinstance(cancel_result, ResponsesAPIResponse) - else: - raise ValueError("response is not a ResponsesAPIResponse") - else: - response = await litellm.aresponses( - input="Basic ping", max_output_tokens=20, background=True, **base_completion_call_args - ) - # async cancel the response - if isinstance(response, ResponsesAPIResponse): - cancel_result = await litellm.acancel_responses( - response_id=response.id, **base_completion_call_args - ) - assert cancel_result is not None - assert hasattr(cancel_result, "id") - # The actual response structure depends on the provider implementation - assert isinstance(cancel_result, ResponsesAPIResponse) + # cancel the response + if isinstance(response, ResponsesAPIResponse): + cancel_result = litellm.cancel_responses( + response_id=response.id, **base_completion_call_args + ) + assert cancel_result is not None + assert hasattr(cancel_result, "id") + # The actual response structure depends on the provider implementation + assert isinstance(cancel_result, ResponsesAPIResponse) + else: + raise ValueError("response is not a ResponsesAPIResponse") else: - raise ValueError("response is not a ResponsesAPIResponse") + response = await litellm.aresponses( + input="Basic ping", max_output_tokens=20, background=True, **base_completion_call_args + ) + + # async cancel the response + if isinstance(response, ResponsesAPIResponse): + cancel_result = await litellm.acancel_responses( + response_id=response.id, **base_completion_call_args + ) + assert cancel_result is not None + assert hasattr(cancel_result, "id") + # The actual response structure depends on the provider implementation + assert isinstance(cancel_result, ResponsesAPIResponse) + else: + raise ValueError("response is not a ResponsesAPIResponse") + except Exception as e: + if "Cannot cancel a completed response" in str(e): + pass + else: + raise e @pytest.mark.parametrize("sync_mode", [False, True]) @pytest.mark.asyncio diff --git a/tests/llm_responses_api_testing/test_anthropic_responses_api.py b/tests/llm_responses_api_testing/test_anthropic_responses_api.py index 8f7a96a016..d633cd0f1d 100644 --- a/tests/llm_responses_api_testing/test_anthropic_responses_api.py +++ b/tests/llm_responses_api_testing/test_anthropic_responses_api.py @@ -34,14 +34,19 @@ class TestAnthropicResponsesAPITest(BaseResponsesAPITest): } async def test_basic_openai_responses_delete_endpoint(self, sync_mode=False): - pass + pytest.skip("DELETE responses is not supported for anthropic") async def test_basic_openai_responses_streaming_delete_endpoint(self, sync_mode=False): - pass + pytest.skip("DELETE responses is not supported for anthropic") async def test_basic_openai_responses_get_endpoint(self, sync_mode=False): - pass - + pytest.skip("GET responses is not supported for anthropic") + + async def test_basic_openai_responses_cancel_endpoint(self, sync_mode=False): + pytest.skip("CANCEL responses is not supported for anthropic") + + async def test_cancel_responses_invalid_response_id(self, sync_mode=False): + pytest.skip("CANCEL responses is not supported for anthropic") diff --git a/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py b/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py index 81daaea238..203ee252b3 100644 --- a/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py +++ b/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py @@ -93,13 +93,20 @@ class TestGoogleAIStudioResponsesAPITest(BaseResponsesAPITest): } async def test_basic_openai_responses_delete_endpoint(self, sync_mode=False): - pass + pytest.skip("DELETE responses is not supported for Google AI Studio") async def test_basic_openai_responses_streaming_delete_endpoint(self, sync_mode=False): - pass + pytest.skip("DELETE responses is not supported for Google AI Studio") async def test_basic_openai_responses_get_endpoint(self, sync_mode=False): - pass + pytest.skip("GET responses is not supported for Google AI Studio") + + async def test_basic_openai_responses_cancel_endpoint(self, sync_mode=False): + pytest.skip("CANCEL responses is not supported for Google AI Studio") + + async def test_cancel_responses_invalid_response_id(self, sync_mode=False): + pytest.skip("CANCEL responses is not supported for Google AI Studio") + diff --git a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py index 7e7def0ee0..de60881820 100644 --- a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py +++ b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py @@ -131,50 +131,62 @@ def test_anthropic_with_responses_api(): def test_cancel_response(): - client = get_test_client() - from litellm.types.llms.openai import ResponsesAPIResponse - response = client.responses.create( - model="gpt-4o", input="just respond with the word 'ping'", background=True - ) - print("basic response=", response) + try: + client = get_test_client() + from litellm.types.llms.openai import ResponsesAPIResponse + response = client.responses.create( + model="gpt-4o", input="just respond with the word 'ping'", background=True + ) + print("basic response=", response) - # cancel the response - cancel_response = client.responses.cancel(response.id) - print("CANCEL response=", cancel_response) - - # verify cancel response structure - assert hasattr(cancel_response, "id") - # Note: Cancel response returns ResponsesAPIResponse, not DeleteResponseResult - # The actual response structure depends on the provider implementation - assert isinstance(cancel_response, ResponsesAPIResponse) - - -def test_cancel_streaming_response(): - client = get_test_client() - from litellm.types.llms.openai import ResponsesAPIResponse - stream = client.responses.create( - model="gpt-4o", input="just respond with the word 'ping'", stream=True, background=True - ) - - collected_chunks = [] - response_id = None - for chunk in stream: - print("stream chunk=", chunk) - collected_chunks.append(chunk) - # Extract response ID from the first chunk that has it - if response_id is None and hasattr(chunk, 'response') and hasattr(chunk.response, 'id'): - response_id = chunk.response.id - - assert len(collected_chunks) > 0 - - # cancel the response if we got a response ID - if response_id: - cancel_response = client.responses.cancel(response_id) - print("CANCEL streaming response=", cancel_response) + # cancel the response + cancel_response = client.responses.cancel(response.id) + print("CANCEL response=", cancel_response) + + # verify cancel response structure assert hasattr(cancel_response, "id") # Note: Cancel response returns ResponsesAPIResponse, not DeleteResponseResult # The actual response structure depends on the provider implementation assert isinstance(cancel_response, ResponsesAPIResponse) + except Exception as e: + if "Cannot cancel a completed response" in str(e): + pass + else: + raise e + + +def test_cancel_streaming_response(): + try: + client = get_test_client() + from litellm.types.llms.openai import ResponsesAPIResponse + stream = client.responses.create( + model="gpt-4o", input="just respond with the word 'ping'", stream=True, background=True + ) + + collected_chunks = [] + response_id = None + for chunk in stream: + print("stream chunk=", chunk) + collected_chunks.append(chunk) + # Extract response ID from the first chunk that has it + if response_id is None and hasattr(chunk, 'response') and hasattr(chunk.response, 'id'): + response_id = chunk.response.id + + assert len(collected_chunks) > 0 + + # cancel the response if we got a response ID + if response_id: + cancel_response = client.responses.cancel(response_id) + print("CANCEL streaming response=", cancel_response) + assert hasattr(cancel_response, "id") + # Note: Cancel response returns ResponsesAPIResponse, not DeleteResponseResult + # The actual response structure depends on the provider implementation + assert isinstance(cancel_response, ResponsesAPIResponse) + except Exception as e: + if "Cannot cancel a completed response" in str(e): + pass + else: + raise e def test_cancel_invalid_response_id(): diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index 5cbd602268..230e251a5d 100644 --- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -971,8 +971,9 @@ async def test_create_group_with_nonexistent_users_creates_users(mocker): # Mock created users return values def mock_new_user_side_effect(data): - from litellm.proxy._types import LiteLLM_UserTable - return LiteLLM_UserTable( + from litellm.proxy._types import NewUserResponse + return NewUserResponse( + key="sk-test-key-" + data.user_id, # Required field from GenerateKeyResponse user_id=data.user_id, user_email=data.user_email, metadata=data.metadata, @@ -1121,8 +1122,9 @@ async def test_update_group_with_nonexistent_users_creates_users(mocker): # Mock created users return values def mock_new_user_side_effect(data): - from litellm.proxy._types import LiteLLM_UserTable - return LiteLLM_UserTable( + from litellm.proxy._types import NewUserResponse + return NewUserResponse( + key="sk-test-key-" + data.user_id, # Required field from GenerateKeyResponse user_id=data.user_id, user_email=data.user_email, metadata=data.metadata,