Fix async retryer on .acompletion() + forward clientside headers - filter out content-type from clientside request (causes llm api call to hang) (#12886)

* fix(main.py): fix async retryer

Fixes https://github.com/BerriAI/litellm/issues/12830

* fix(forward_clientside_headers_by_model_group.py): filter out 'content-type' from forwardable headers

clientside content-type != proxy content type, can cause requests to hang

litellm/main.py

@@ -3483,13 +3483,13 @@ async def acompletion_with_retries(*args, **kwargs):
     retry_strategy = kwargs.pop("retry_strategy", "constant_retry")
     original_function = kwargs.pop("original_function", completion)
     if retry_strategy == "exponential_backoff_retry":
-        retryer = tenacity.Retrying(
+        retryer = tenacity.AsyncRetrying(
             wait=tenacity.wait_exponential(multiplier=1, max=10),
             stop=tenacity.stop_after_attempt(num_retries),
             reraise=True,
         )
     else:
-        retryer = tenacity.Retrying(
+        retryer = tenacity.AsyncRetrying(
             stop=tenacity.stop_after_attempt(num_retries), reraise=True
         )
     return await retryer(original_function, *args, **kwargs)

litellm/proxy/_new_secret_config.yaml

@@ -6,6 +6,22 @@ model_list:
       api_base: os.environ/AZURE_API_BASE_HIDDEN
     model_info: 
       version: 2
+  - model_name: gpt-3.5-turbo-disallow
+    litellm_params:
+      model: gpt-3.5-turbo
+    model_info: 
+      version: 2
+  - model_name: zapier-byok-provider/openai/*
+    litellm_params:
+      model: openai/*
+      api_base: http://0.0.0.0:8090
+  - model_name: openai/gpt-4o-mini
+    litellm_params:
+      model: openai/gpt-4o-mini
 
-general_settings:
-  proxy_batch_polling_interval: 10
+litellm_settings:
+  model_group_alias: {"gpt-3.5-turbo-custom": "gpt-3.5-turbo-disallow"}
+  model_group_settings:
+    forward_client_headers_to_llm_api: 
+      - "gpt-3.5-turbo-allow"
+      - "zapier-byok-provider/openai/*"

litellm/router_utils/forward_clientside_headers_by_model_group.py

@@ -31,6 +31,18 @@ class ForwardClientSideHeadersByModelGroup(CustomLogger):
             "model_group_alias": model_group_alias,
         }
 
+    def filter_headers(self, headers: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Filter the headers to only include the headers that are forwarded to the LLM API.
+
+        E.g. passing 'connection': 'keep-alive' will cause the request to hang, and not be acknowledged on the other side.
+        """
+        return {
+            k: v
+            for k, v in headers.items()
+            if k.lower() not in ["connection", "content-length"]
+        }
+
     async def async_pre_call_deployment_hook(
         self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
     ) -> Optional[dict]:
@@ -66,7 +78,7 @@ class ForwardClientSideHeadersByModelGroup(CustomLogger):
                     in litellm.model_group_settings.forward_client_headers_to_llm_api
                 ):
                     kwargs.setdefault("headers", {}).update(
-                        kwargs["secret_fields"]["raw_headers"]
+                        self.filter_headers(kwargs["secret_fields"]["raw_headers"])
                     )
 
         return kwargs

tests/test_litellm/test_main.py

@@ -1097,3 +1097,28 @@ def test_stream_chunk_builder_thinking_blocks():
     assert response is not None
     assert response.choices[0].message.content is not None
     assert response.choices[0].message.thinking_blocks is not None
+
+
+from litellm.llms.openai.openai import OpenAIChatCompletion
+
+
+def throw_retryable_error(*_, **__):
+    raise RuntimeError("BOOM")
+
+
+@pytest.mark.asyncio
+async def test_retrying() -> None:
+    litellm.num_retries = 10
+    with (
+        patch.object(
+            OpenAIChatCompletion,
+            "make_openai_chat_completion_request",
+            side_effect=throw_retryable_error,
+        ) as mock_request,
+        pytest.raises(litellm.InternalServerError, match="LiteLLM Retried: 10 times"),
+    ):
+        await litellm.acompletion(
+            model="gpt-4o-mini",
+            messages=[{"role": "user", "content": "Hello"}],
+        )
+    assert mock_request.call_count >= 10, "Expected retrying to be used"