From dfb68a23de435afc4c2503c1caf552f1a116f154 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 9 Jun 2026 02:27:03 +0530 Subject: [PATCH] feat(galileo): add health check support for UI callback test (#29908) * feat(galileo): add health check support for UI callback test Register galileo in /health/services so the proxy UI callback connection test works. Co-authored-by: Cursor * feat(galileo): verify API key via /current_user health check Call Galileo's current_user endpoint so the UI callback test validates credentials against the provider. Co-authored-by: Cursor * chore(ui): regenerate schema.d.ts for galileo health service Co-authored-by: Cursor * fix(galileo): return IntegrationHealthCheckStatus from async_health_check Fixes mypy assignment error in health_services_endpoint where response was narrowed to IntegrationHealthCheckStatus from earlier branches. Co-authored-by: Cursor * Fix Galileo logging to match Langfuse across all endpoint types. Stop skipping ingest when output is empty and log embeddings with a placeholder so embedding, speech, and other non-text responses are recorded like Langfuse. Co-authored-by: Cursor * fix(galileo): remove unreachable health-check guard and None output sentinel The use_v2_api flag is derived from bool(api_key), so the inner GALILEO_API_KEY check inside the v2 branch could never run; collapse the credential validation into the username/password path with a combined message. _serialize_galileo_output now returns an empty string for None, so _get_galileo_input_output_content always yields a str and the post-call None coalescing guard is no longer needed. * test(galileo): cover async_health_check failure paths and empty model response Add regression tests for the Galileo health check unhealthy branches (missing project id, missing base url, missing credentials, auth failure, and request exception) and for logging a model response with no choices, which now queues an empty output instead of being skipped. --------- Co-authored-by: Cursor Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/integrations/galileo.py | 71 ++++-- .../health_endpoints/_health_endpoints.py | 15 ++ .../test_litellm/integrations/test_galileo.py | 206 +++++++++++++++++- .../health_endpoints/test_health_endpoints.py | 27 +++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 5 files changed, 303 insertions(+), 18 deletions(-) diff --git a/litellm/integrations/galileo.py b/litellm/integrations/galileo.py index 8fef90c24e..f9ff7e8c7a 100644 --- a/litellm/integrations/galileo.py +++ b/litellm/integrations/galileo.py @@ -26,6 +26,7 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) +from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus GALILEO_CLOUD_API_BASE_URL = "https://api.galileo.ai" # Cap the in-memory buffer so persistent flush failures (e.g. Galileo @@ -89,6 +90,52 @@ class GalileoObserve(CustomLogger): return bool(self.api_key) return bool(self.username and self.password) + async def async_health_check(self) -> IntegrationHealthCheckStatus: + try: + if not self.project_id: + return IntegrationHealthCheckStatus( + status="unhealthy", + error_message="GALILEO_PROJECT_ID environment variable not set", + ) + + if not self.base_url: + return IntegrationHealthCheckStatus( + status="unhealthy", + error_message="GALILEO_BASE_URL environment variable not set", + ) + + if not self.use_v2_api and (not self.username or not self.password): + return IntegrationHealthCheckStatus( + status="unhealthy", + error_message=( + "GALILEO_API_KEY or GALILEO_USERNAME and GALILEO_PASSWORD " + "environment variables must be set" + ), + ) + + if not await self._ensure_headers(): + return IntegrationHealthCheckStatus( + status="unhealthy", + error_message="Galileo authentication failed", + ) + + response = await self.async_httpx_handler.get( + url=f"{self.base_url}/current_user", + headers=self.headers, + ) + if response.status_code >= 400: + return IntegrationHealthCheckStatus( + status="unhealthy", + error_message=(f"Galileo API returned HTTP {response.status_code}"), + ) + + return IntegrationHealthCheckStatus(status="healthy", error_message=None) + except Exception as e: + return IntegrationHealthCheckStatus( + status="unhealthy", + error_message=f"Galileo health check failed: {str(e)}", + ) + async def async_set_galileo_headers(self) -> None: galileo_login_response = await self.async_httpx_handler.post( url=f"{self.base_url}/login", @@ -399,9 +446,9 @@ class GalileoObserve(CustomLogger): return prompt @staticmethod - def _serialize_galileo_output(value: Any) -> Optional[str]: + def _serialize_galileo_output(value: Any) -> str: if value is None: - return None + return "" if isinstance(value, str): return value @@ -460,11 +507,11 @@ class GalileoObserve(CustomLogger): response_obj: Any, level: str = "DEFAULT", status_message: Optional[str] = None, - ) -> Tuple[str, Optional[str], Any]: + ) -> Tuple[str, str, Any]: """ Mirror Langfuse _get_langfuse_input_output_content for Galileo ingest. - Returns (input_text, output_text, messages_for_span). output_text None skips ingest. + Returns (input_text, output_text, messages_for_span). """ call_type = kwargs.get("call_type") prompt = self._build_prompt(kwargs) @@ -477,10 +524,11 @@ class GalileoObserve(CustomLogger): return self._prompt_to_input_text(prompt), status_message, prompt if response_obj is not None and ( - call_type == "embedding" + call_type in ("embedding", "aembedding") or isinstance(response_obj, litellm.EmbeddingResponse) ): - return self._prompt_to_input_text(prompt), None, prompt + # Match Langfuse OTEL: log embeddings without serializing vectors. + return self._prompt_to_input_text(prompt), "embedding-output", prompt if response_obj is not None and isinstance(response_obj, litellm.ModelResponse): output = self._get_chat_content_for_galileo(response_obj) @@ -549,7 +597,7 @@ class GalileoObserve(CustomLogger): ): input_val = kwargs.get("input") return ( - self._serialize_galileo_output(input_val) or "", + self._serialize_galileo_output(input_val), self._serialize_galileo_output(response_obj), input_val, ) @@ -574,11 +622,11 @@ class GalileoObserve(CustomLogger): kwargs.get("messages") or [], ) - return self._prompt_to_input_text(prompt), None, kwargs.get("messages") or [] + return self._prompt_to_input_text(prompt), "", kwargs.get("messages") or [] def get_output_str_from_response( self, response_obj: Any, kwargs: Dict[str, Any] - ) -> Optional[str]: + ) -> str: _, output_text, _ = self._get_galileo_input_output_content( kwargs=kwargs, response_obj=response_obj ) @@ -659,11 +707,6 @@ class GalileoObserve(CustomLogger): input_text, output_text, messages = self._get_galileo_input_output_content( kwargs=kwargs, response_obj=response_obj ) - if output_text is None: - verbose_logger.debug( - "Galileo Logger: skipping %s — no text output to log", _call_type - ) - return raw_start = slo.get("startTime") raw_end = slo.get("endTime") diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index c109f37499..6ef8bbc400 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -129,6 +129,7 @@ services = Union[ "datadog_llm_observability", "generic_api", "arize", + "galileo", "sqs", ], str, @@ -206,6 +207,7 @@ async def health_services_endpoint( # noqa: PLR0915 "datadog_llm_observability", "generic_api", "arize", + "galileo", "sqs", ]: raise HTTPException( @@ -295,6 +297,19 @@ async def health_services_endpoint( # noqa: PLR0915 else "Arize is healthy" ), } + elif service == "galileo": + from litellm.integrations.galileo import GalileoObserve + + galileo_logger = GalileoObserve() + response = await galileo_logger.async_health_check() + return { + "status": response["status"], + "message": ( + response["error_message"] + if response["status"] == "unhealthy" + else "Galileo is healthy" + ), + } elif service == "langfuse": from litellm.integrations.langfuse.langfuse import LangFuseLogger diff --git a/tests/test_litellm/integrations/test_galileo.py b/tests/test_litellm/integrations/test_galileo.py index 8ce5eb776f..0533b7ca7d 100644 --- a/tests/test_litellm/integrations/test_galileo.py +++ b/tests/test_litellm/integrations/test_galileo.py @@ -357,12 +357,18 @@ def test_galileo_record_to_v2_span_with_tags_and_offset(): def test_galileo_get_output_str_variants(galileo_v2_env): logger = GalileoObserve() - assert logger.get_output_str_from_response(None, {}) is None + assert logger.get_output_str_from_response(None, {}) == "" assert ( logger.get_output_str_from_response( EmbeddingResponse(), {"call_type": "embedding"} ) - is None + == "embedding-output" + ) + assert ( + logger.get_output_str_from_response( + EmbeddingResponse(), {"call_type": "aembedding"} + ) + == "embedding-output" ) text_resp = TextCompletionResponse() @@ -414,7 +420,7 @@ def test_galileo_get_output_str_variants(galileo_v2_env): {"call_type": "acompletion", "messages": [{"role": "user", "content": "hi"}]}, ) - assert logger.get_output_str_from_response("not-a-supported-type", {}) is None + assert logger.get_output_str_from_response("not-a-supported-type", {}) == "" def test_galileo_get_input_output_error_status_message(galileo_v2_env): @@ -445,6 +451,48 @@ def test_galileo_get_output_str_rerank_response(galileo_v2_env): assert '"relevance_score": 0.98' in output +@pytest.mark.asyncio +async def test_galileo_async_log_success_embedding(galileo_v2_env): + import datetime + + logger = GalileoObserve() + embedding_response = EmbeddingResponse( + data=[{"object": "embedding", "embedding": [0.1, 0.2, 0.3], "index": 0}] + ) + + mock_response = MagicMock() + mock_response.is_success = True + mock_response.status_code = 201 + + with patch.object(logger.async_httpx_handler, "post", return_value=mock_response): + await logger.async_log_success_event( + kwargs={ + "call_type": "aembedding", + "model": "text-embedding-3-small", + "input": "hello world", + "standard_logging_object": { + "call_type": "aembedding", + "model": "text-embedding-3-small", + "prompt_tokens": 2, + "completion_tokens": 0, + "total_tokens": 2, + "response_cost": 0.0, + "startTime": datetime.datetime( + 2026, 5, 25, 12, 0, 0, tzinfo=datetime.timezone.utc + ).timestamp(), + "endTime": datetime.datetime( + 2026, 5, 25, 12, 0, 1, tzinfo=datetime.timezone.utc + ).timestamp(), + }, + }, + response_obj=embedding_response, + start_time=datetime.datetime(2026, 5, 25, 12, 0, 0), + end_time=datetime.datetime(2026, 5, 25, 12, 0, 1), + ) + + assert logger.in_memory_records == [] + + @pytest.mark.asyncio async def test_galileo_async_log_success_rerank(galileo_v2_env): import datetime @@ -524,6 +572,158 @@ def test_galileo_get_ingest_request_legacy(monkeypatch): assert payload["traces"][0]["input"] == "hi" +@pytest.mark.asyncio +async def test_galileo_async_health_check_success(galileo_v2_env): + logger = GalileoObserve() + current_user_resp = MagicMock() + current_user_resp.status_code = 200 + + with patch.object( + logger.async_httpx_handler, "get", new_callable=AsyncMock + ) as mock_get: + mock_get.return_value = current_user_resp + result = await logger.async_health_check() + + assert result["status"] == "healthy" + mock_get.assert_awaited_once_with( + url="https://api.galileo.ai/current_user", + headers={ + "accept": "application/json", + "Content-Type": "application/json", + "Galileo-API-Key": "test-api-key", + }, + ) + + +@pytest.mark.asyncio +async def test_galileo_async_health_check_api_error(galileo_v2_env): + logger = GalileoObserve() + current_user_resp = MagicMock() + current_user_resp.status_code = 401 + + with patch.object( + logger.async_httpx_handler, "get", new_callable=AsyncMock + ) as mock_get: + mock_get.return_value = current_user_resp + result = await logger.async_health_check() + + assert result["status"] == "unhealthy" + assert "HTTP 401" in result["error_message"] + + +@pytest.mark.asyncio +async def test_galileo_async_health_check_missing_project_id(monkeypatch): + monkeypatch.setenv("GALILEO_API_KEY", "test-api-key") + monkeypatch.setenv("GALILEO_BASE_URL", "https://api.galileo.ai") + monkeypatch.delenv("GALILEO_PROJECT_ID", raising=False) + logger = GalileoObserve() + + result = await logger.async_health_check() + + assert result["status"] == "unhealthy" + assert "GALILEO_PROJECT_ID" in result["error_message"] + + +@pytest.mark.asyncio +async def test_galileo_async_health_check_missing_base_url(monkeypatch): + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.delenv("GALILEO_BASE_URL", raising=False) + monkeypatch.setenv("GALILEO_PROJECT_ID", "p") + monkeypatch.setenv("GALILEO_USERNAME", "u") + monkeypatch.setenv("GALILEO_PASSWORD", "pw") + logger = GalileoObserve() + + result = await logger.async_health_check() + + assert result["status"] == "unhealthy" + assert "GALILEO_BASE_URL" in result["error_message"] + + +@pytest.mark.asyncio +async def test_galileo_async_health_check_missing_credentials(monkeypatch): + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.delenv("GALILEO_USERNAME", raising=False) + monkeypatch.delenv("GALILEO_PASSWORD", raising=False) + monkeypatch.setenv("GALILEO_PROJECT_ID", "p") + monkeypatch.setenv("GALILEO_BASE_URL", "https://galileo.example") + logger = GalileoObserve() + + result = await logger.async_health_check() + + assert result["status"] == "unhealthy" + assert "GALILEO_USERNAME" in result["error_message"] + + +@pytest.mark.asyncio +async def test_galileo_async_health_check_auth_failed(monkeypatch): + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.setenv("GALILEO_PROJECT_ID", "p") + monkeypatch.setenv("GALILEO_BASE_URL", "https://galileo.example") + monkeypatch.setenv("GALILEO_USERNAME", "u") + monkeypatch.setenv("GALILEO_PASSWORD", "pw") + logger = GalileoObserve() + + with patch.object( + logger.async_httpx_handler, "post", new_callable=AsyncMock + ) as mock_post: + mock_post.side_effect = Exception("login failed") + result = await logger.async_health_check() + + assert result["status"] == "unhealthy" + assert result["error_message"] == "Galileo authentication failed" + + +@pytest.mark.asyncio +async def test_galileo_async_health_check_request_exception(galileo_v2_env): + logger = GalileoObserve() + + with patch.object( + logger.async_httpx_handler, "get", new_callable=AsyncMock + ) as mock_get: + mock_get.side_effect = Exception("connection refused") + result = await logger.async_health_check() + + assert result["status"] == "unhealthy" + assert "connection refused" in result["error_message"] + + +@pytest.mark.asyncio +async def test_galileo_async_log_success_empty_model_response(galileo_v2_env): + import datetime + + logger = GalileoObserve() + logger.batch_size = 2 + empty_response = ModelResponse(choices=[]) + + await logger.async_log_success_event( + kwargs={ + "call_type": "acompletion", + "model": "gpt-5.2", + "messages": [{"role": "user", "content": "hi"}], + "standard_logging_object": { + "call_type": "acompletion", + "model": "gpt-5.2", + "prompt_tokens": 1, + "completion_tokens": 0, + "total_tokens": 1, + "response_cost": 0.0, + "startTime": datetime.datetime( + 2026, 5, 25, 12, 0, 0, tzinfo=datetime.timezone.utc + ).timestamp(), + "endTime": datetime.datetime( + 2026, 5, 25, 12, 0, 1, tzinfo=datetime.timezone.utc + ).timestamp(), + }, + }, + response_obj=empty_response, + start_time=datetime.datetime(2026, 5, 25, 12, 0, 0), + end_time=datetime.datetime(2026, 5, 25, 12, 0, 1), + ) + + assert len(logger.in_memory_records) == 1 + assert logger.in_memory_records[0]["output_text"] == "" + + @pytest.mark.asyncio async def test_galileo_ensure_headers_v2_missing_key(monkeypatch): monkeypatch.delenv("GALILEO_API_KEY", raising=False) diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index 80a4804956..64c57ab90e 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -696,6 +696,33 @@ async def test_test_model_connection_falls_back_to_deployments_zero_without_id() assert model_params.get("api_key") == "fake-key-A" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "status,error_message", + [ + ("healthy", ""), + ("unhealthy", "Galileo authentication failed"), + ], +) +async def test_health_services_endpoint_galileo(status, error_message): + with patch("litellm.integrations.galileo.GalileoObserve") as MockGalileoObserve: + mock_instance = MagicMock() + mock_instance.async_health_check = AsyncMock( + return_value={"status": status, "error_message": error_message} + ) + MockGalileoObserve.return_value = mock_instance + + result = await health_services_endpoint(service="galileo") + + if status == "healthy": + assert result["status"] == "healthy" + assert result["message"] == "Galileo is healthy" + else: + assert result["status"] == "unhealthy" + assert result["message"] == error_message + mock_instance.async_health_check.assert_awaited_once() + + @pytest.mark.asyncio async def test_health_services_endpoint_datadog_llm_observability(): """ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d9992680c8..8379d0536a 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -40422,7 +40422,7 @@ export interface operations { parameters: { query: { /** @description Specify the service being hit. */ - service: ("slack_budget_alerts" | "langfuse" | "langfuse_otel" | "slack" | "openmeter" | "webhook" | "email" | "braintrust" | "datadog" | "datadog_llm_observability" | "generic_api" | "arize" | "sqs") | string; + service: ("slack_budget_alerts" | "langfuse" | "langfuse_otel" | "slack" | "openmeter" | "webhook" | "email" | "braintrust" | "datadog" | "datadog_llm_observability" | "generic_api" | "arize" | "galileo" | "sqs") | string; }; header?: never; path?: never;