ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Fix] /responses API - add cancel endpoint + allow non-admins to use this as an llm api endpoint (#14594)

Browse Source 
* fix: ensure /responses/cancel works for non admins

* test: cancel endpoint

* fix responses API  cancel endpoint

* test fix

* TestGoogleAIStudioResponsesAPITest
This commit is contained in:
Ishaan Jaff 2025-09-15 18:49:54 -07:00 committed by GitHub
parent e1a6b9f858
commit 8e22cf5d65
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 116 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
litellm/proxy/_types.py
View File

@ -312,6 +312,8 @@ class LiteLLMRoutes(enum.Enum):
"/v1/responses/{response_id}",
"/responses/{response_id}/input_items",
"/v1/responses/{response_id}/input_items",
"/responses/{response_id}/cancel",
"/v1/responses/{response_id}/cancel",
# vector stores
"/vector_stores",
"/v1/vector_stores",

70
tests/llm_responses_api_testing/base_responses_api.py
View File

@ -595,41 +595,47 @@ class BaseResponsesAPITest(ABC):
@pytest.mark.flaky(retries=3, delay=2)
@pytest.mark.asyncio
async def test_basic_openai_responses_cancel_endpoint(self, sync_mode):
litellm._turn_on_debug()
litellm.set_verbose = True
base_completion_call_args = self.get_base_completion_call_args()
if sync_mode:
response = litellm.responses(
input="Basic ping", max_output_tokens=20, background=True, **base_completion_call_args
)
# cancel the response
if isinstance(response, ResponsesAPIResponse):
cancel_result = litellm.cancel_responses(
response_id=response.id, **base_completion_call_args
try:
litellm._turn_on_debug()
litellm.set_verbose = True
base_completion_call_args = self.get_base_completion_call_args()
if sync_mode:
response = litellm.responses(
input="Basic ping", max_output_tokens=20, background=True, **base_completion_call_args
)
assert cancel_result is not None
assert hasattr(cancel_result, "id")
# The actual response structure depends on the provider implementation
assert isinstance(cancel_result, ResponsesAPIResponse)
else:
raise ValueError("response is not a ResponsesAPIResponse")
else:
response = await litellm.aresponses(
input="Basic ping", max_output_tokens=20, background=True, **base_completion_call_args
)
# async cancel the response
if isinstance(response, ResponsesAPIResponse):
cancel_result = await litellm.acancel_responses(
response_id=response.id, **base_completion_call_args
)
assert cancel_result is not None
assert hasattr(cancel_result, "id")
# The actual response structure depends on the provider implementation
assert isinstance(cancel_result, ResponsesAPIResponse)
# cancel the response
if isinstance(response, ResponsesAPIResponse):
cancel_result = litellm.cancel_responses(
response_id=response.id, **base_completion_call_args
)
assert cancel_result is not None
assert hasattr(cancel_result, "id")
# The actual response structure depends on the provider implementation
assert isinstance(cancel_result, ResponsesAPIResponse)
else:
raise ValueError("response is not a ResponsesAPIResponse")
else:
raise ValueError("response is not a ResponsesAPIResponse")
response = await litellm.aresponses(
input="Basic ping", max_output_tokens=20, background=True, **base_completion_call_args
)
# async cancel the response
if isinstance(response, ResponsesAPIResponse):
cancel_result = await litellm.acancel_responses(
response_id=response.id, **base_completion_call_args
)
assert cancel_result is not None
assert hasattr(cancel_result, "id")
# The actual response structure depends on the provider implementation
assert isinstance(cancel_result, ResponsesAPIResponse)
else:
raise ValueError("response is not a ResponsesAPIResponse")
except Exception as e:
if "Cannot cancel a completed response" in str(e):
pass
else:
raise e
@pytest.mark.parametrize("sync_mode", [False, True])
@pytest.mark.asyncio

13
tests/llm_responses_api_testing/test_anthropic_responses_api.py
View File

@ -34,14 +34,19 @@ class TestAnthropicResponsesAPITest(BaseResponsesAPITest):
}
async def test_basic_openai_responses_delete_endpoint(self, sync_mode=False):
pass
pytest.skip("DELETE responses is not supported for anthropic")
async def test_basic_openai_responses_streaming_delete_endpoint(self, sync_mode=False):
pass
pytest.skip("DELETE responses is not supported for anthropic")
async def test_basic_openai_responses_get_endpoint(self, sync_mode=False):
pass
pytest.skip("GET responses is not supported for anthropic")
async def test_basic_openai_responses_cancel_endpoint(self, sync_mode=False):
pytest.skip("CANCEL responses is not supported for anthropic")
async def test_cancel_responses_invalid_response_id(self, sync_mode=False):
pytest.skip("CANCEL responses is not supported for anthropic")

13
tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py
View File

@ -93,13 +93,20 @@ class TestGoogleAIStudioResponsesAPITest(BaseResponsesAPITest):
}
async def test_basic_openai_responses_delete_endpoint(self, sync_mode=False):
pass
pytest.skip("DELETE responses is not supported for Google AI Studio")
async def test_basic_openai_responses_streaming_delete_endpoint(self, sync_mode=False):
pass
pytest.skip("DELETE responses is not supported for Google AI Studio")
async def test_basic_openai_responses_get_endpoint(self, sync_mode=False):
pass
pytest.skip("GET responses is not supported for Google AI Studio")
async def test_basic_openai_responses_cancel_endpoint(self, sync_mode=False):
pytest.skip("CANCEL responses is not supported for Google AI Studio")
async def test_cancel_responses_invalid_response_id(self, sync_mode=False):
pytest.skip("CANCEL responses is not supported for Google AI Studio")

90
tests/openai_endpoints_tests/test_e2e_openai_responses_api.py
View File

@ -131,50 +131,62 @@ def test_anthropic_with_responses_api():
def test_cancel_response():
client = get_test_client()
from litellm.types.llms.openai import ResponsesAPIResponse
response = client.responses.create(
model="gpt-4o", input="just respond with the word 'ping'", background=True
)
print("basic response=", response)
try:
client = get_test_client()
from litellm.types.llms.openai import ResponsesAPIResponse
response = client.responses.create(
model="gpt-4o", input="just respond with the word 'ping'", background=True
)
print("basic response=", response)
# cancel the response
cancel_response = client.responses.cancel(response.id)
print("CANCEL response=", cancel_response)
# verify cancel response structure
assert hasattr(cancel_response, "id")
# Note: Cancel response returns ResponsesAPIResponse, not DeleteResponseResult
# The actual response structure depends on the provider implementation
assert isinstance(cancel_response, ResponsesAPIResponse)
def test_cancel_streaming_response():
client = get_test_client()
from litellm.types.llms.openai import ResponsesAPIResponse
stream = client.responses.create(
model="gpt-4o", input="just respond with the word 'ping'", stream=True, background=True
)
collected_chunks = []
response_id = None
for chunk in stream:
print("stream chunk=", chunk)
collected_chunks.append(chunk)
# Extract response ID from the first chunk that has it
if response_id is None and hasattr(chunk, 'response') and hasattr(chunk.response, 'id'):
response_id = chunk.response.id
assert len(collected_chunks) > 0
# cancel the response if we got a response ID
if response_id:
cancel_response = client.responses.cancel(response_id)
print("CANCEL streaming response=", cancel_response)
# cancel the response
cancel_response = client.responses.cancel(response.id)
print("CANCEL response=", cancel_response)
# verify cancel response structure
assert hasattr(cancel_response, "id")
# Note: Cancel response returns ResponsesAPIResponse, not DeleteResponseResult
# The actual response structure depends on the provider implementation
assert isinstance(cancel_response, ResponsesAPIResponse)
except Exception as e:
if "Cannot cancel a completed response" in str(e):
pass
else:
raise e
def test_cancel_streaming_response():
try:
client = get_test_client()
from litellm.types.llms.openai import ResponsesAPIResponse
stream = client.responses.create(
model="gpt-4o", input="just respond with the word 'ping'", stream=True, background=True
)
collected_chunks = []
response_id = None
for chunk in stream:
print("stream chunk=", chunk)
collected_chunks.append(chunk)
# Extract response ID from the first chunk that has it
if response_id is None and hasattr(chunk, 'response') and hasattr(chunk.response, 'id'):
response_id = chunk.response.id
assert len(collected_chunks) > 0
# cancel the response if we got a response ID
if response_id:
cancel_response = client.responses.cancel(response_id)
print("CANCEL streaming response=", cancel_response)
assert hasattr(cancel_response, "id")
# Note: Cancel response returns ResponsesAPIResponse, not DeleteResponseResult
# The actual response structure depends on the provider implementation
assert isinstance(cancel_response, ResponsesAPIResponse)
except Exception as e:
if "Cannot cancel a completed response" in str(e):
pass
else:
raise e
def test_cancel_invalid_response_id():

10
tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py
View File

@ -971,8 +971,9 @@ async def test_create_group_with_nonexistent_users_creates_users(mocker):
# Mock created users return values
def mock_new_user_side_effect(data):
from litellm.proxy._types import LiteLLM_UserTable
return LiteLLM_UserTable(
from litellm.proxy._types import NewUserResponse
return NewUserResponse(
key="sk-test-key-" + data.user_id, # Required field from GenerateKeyResponse
user_id=data.user_id,
user_email=data.user_email,
metadata=data.metadata,
@ -1121,8 +1122,9 @@ async def test_update_group_with_nonexistent_users_creates_users(mocker):
# Mock created users return values
def mock_new_user_side_effect(data):
from litellm.proxy._types import LiteLLM_UserTable
return LiteLLM_UserTable(
from litellm.proxy._types import NewUserResponse
return NewUserResponse(
key="sk-test-key-" + data.user_id, # Required field from GenerateKeyResponse
user_id=data.user_id,
user_email=data.user_email,
metadata=data.metadata,