fix(proxy): disable proxy buffering on streaming SSE responses (#29557)

Streaming responses from the proxy (/chat/completions, /v1/messages,
/v1/responses, assistants) all return through create_response() but never
sent the headers that tell an intermediary reverse proxy not to buffer the
SSE stream. nginx with the default proxy_buffering, k8s ingress-nginx, and
Envoy/Istio sidecars therefore hold the whole stream and release it in one
batch, which looks like a broken/buffered stream to the client even though
litellm is yielding chunks incrementally.

Add Cache-Control: no-cache and X-Accel-Buffering: no to every
StreamingResponse create_response() returns, matching what the proxy already
does for its own usage/policy SSE endpoints. Fixes #28384.

litellm/proxy/common_request_processing.py

@@ -249,6 +249,13 @@ async def create_response(  # noqa: PLR0915
     If the first chunk is an error, return a standard JSON error response.
     Otherwise, return StreamingResponse and stream all content.
     """
+    # Tell buffering reverse proxies (nginx, ingress-nginx, Envoy) to flush SSE
+    # immediately instead of releasing the whole stream in one batch (issue #28384).
+    streaming_headers = {
+        **headers,
+        "Cache-Control": "no-cache",
+        "X-Accel-Buffering": "no",
+    }
     first_chunk_value: Optional[str] = None
     final_status_code = default_status_code
 
@@ -300,7 +307,7 @@ async def create_response(  # noqa: PLR0915
         return StreamingResponse(
             empty_gen(),
             media_type=media_type,
-            headers=headers,
+            headers=streaming_headers,
             status_code=default_status_code,
         )
     except Exception as e:
@@ -338,7 +345,7 @@ async def create_response(  # noqa: PLR0915
         return StreamingResponse(
             error_gen_message(),
             media_type=media_type,
-            headers=headers,
+            headers=streaming_headers,
             status_code=error_status,
         )
 
@@ -360,7 +367,7 @@ async def create_response(  # noqa: PLR0915
     return StreamingResponse(
         combined_generator(),
         media_type=media_type,
-        headers=headers,
+        headers=streaming_headers,
         status_code=final_status_code,
     )
 

tests/test_litellm/proxy/test_common_request_processing.py

@@ -1258,6 +1258,31 @@ class TestCommonRequestProcessingHelpers:
         )
         assert response.headers["x-custom-header"] == "TestValue"
 
+    async def test_create_streaming_response_disables_proxy_buffering(self):
+        """Regression for #28384: every StreamingResponse create_response returns
+        must carry the headers that stop nginx/ingress/Envoy from buffering the
+        SSE stream into one batch, while preserving caller-supplied headers."""
+
+        async def normal_stream():
+            yield 'data: {"content": "part"}\n\n'
+            yield "data: [DONE]\n\n"
+
+        async def empty_stream():
+            if False:  # never yields -> StopAsyncIteration
+                yield
+
+        error_stream = AsyncMock()
+        error_stream.__anext__.side_effect = ValueError("boom")
+
+        for generator in (normal_stream(), empty_stream(), error_stream):
+            response = await create_response(
+                generator, "text/event-stream", {"X-Custom-Header": "keep"}
+            )
+            assert isinstance(response, StreamingResponse)
+            assert response.headers["x-accel-buffering"] == "no"
+            assert response.headers["cache-control"] == "no-cache"
+            assert response.headers["x-custom-header"] == "keep"
+
     async def test_create_streaming_response_non_default_status_code(self):
         async def mock_generator():
             yield 'data: {"content": "data"}\n\n'