From 2d5ae35a8517aa480049d37600a531d0eb6d85b8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 6 Nov 2025 12:38:47 -0800 Subject: [PATCH] Show all callbacks on UI --- litellm/proxy/proxy_server.py | 123 ++++--- tests/proxy_unit_tests/test_proxy_server.py | 163 ++++++++ .../src/components/settings.test.tsx | 136 +++++++ .../src/components/settings.tsx | 348 ++++++++++-------- 4 files changed, 548 insertions(+), 222 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/settings.test.tsx diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bed3d218df..4a0d39f451 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9572,8 +9572,61 @@ async def get_config(): # noqa: PLR0915 _general_settings = config_data.get("general_settings", {}) environment_variables = config_data.get("environment_variables", {}) - # check if "langfuse" in litellm_settings + # Helper function to process callbacks and get environment variables + def process_callback(_callback: str, callback_type: str) -> dict: + """Process a single callback and return its data with environment variables""" + if _callback == "langfuse" or _callback == "langfuse_otel": + env_vars = [ + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + "LANGFUSE_HOST", + ] + elif _callback == "openmeter": + env_vars = [ + "OPENMETER_API_KEY", + ] + elif _callback == "braintrust": + env_vars = [ + "BRAINTRUST_API_KEY", + "BRAINTRUST_API_BASE", + ] + elif _callback == "traceloop": + env_vars = ["TRACELOOP_API_KEY"] + elif _callback == "custom_callback_api": + env_vars = ["GENERIC_LOGGER_ENDPOINT"] + elif _callback == "otel": + env_vars = ["OTEL_EXPORTER", "OTEL_ENDPOINT", "OTEL_HEADERS"] + elif _callback == "langsmith": + env_vars = [ + "LANGSMITH_API_KEY", + "LANGSMITH_PROJECT", + "LANGSMITH_DEFAULT_RUN_NAME", + ] + else: + env_vars = [] + + env_vars_dict = {} + for _var in env_vars: + env_variable = environment_variables.get(_var, None) + if env_variable is None: + env_vars_dict[_var] = None + else: + # decode + decrypt the value + decrypted_value = decrypt_value_helper( + value=env_variable, key=_var + ) + env_vars_dict[_var] = decrypted_value + + return { + "name": _callback, + "variables": env_vars_dict, + "type": callback_type + } + _success_callbacks = _litellm_settings.get("success_callback", []) + _failure_callbacks = _litellm_settings.get("failure_callback", []) + _generic_callbacks = _litellm_settings.get("callbacks", []) + _data_to_return = [] """ [ @@ -9584,70 +9637,20 @@ async def get_config(): # noqa: PLR0915 "LANGFUSE_SECRET_KEY": "value", "LANGFUSE_HOST": "value" }, + "type": "success" } ] """ + for _callback in _success_callbacks: - if _callback != "langfuse": - if _callback == "openmeter": - env_vars = [ - "OPENMETER_API_KEY", - ] - elif _callback == "braintrust": - env_vars = [ - "BRAINTRUST_API_KEY", - "BRAINTRUST_API_BASE", - ] - elif _callback == "traceloop": - env_vars = ["TRACELOOP_API_KEY"] - elif _callback == "custom_callback_api": - env_vars = ["GENERIC_LOGGER_ENDPOINT"] - elif _callback == "otel": - env_vars = ["OTEL_EXPORTER", "OTEL_ENDPOINT", "OTEL_HEADERS"] - elif _callback == "langsmith": - env_vars = [ - "LANGSMITH_API_KEY", - "LANGSMITH_PROJECT", - "LANGSMITH_DEFAULT_RUN_NAME", - ] - else: - env_vars = [] - - env_vars_dict = {} - for _var in env_vars: - env_variable = environment_variables.get(_var, None) - if env_variable is None: - env_vars_dict[_var] = None - else: - # decode + decrypt the value - decrypted_value = decrypt_value_helper( - value=env_variable, key=_var - ) - env_vars_dict[_var] = decrypted_value - - _data_to_return.append({"name": _callback, "variables": env_vars_dict}) - elif _callback == "langfuse": - _langfuse_vars = [ - "LANGFUSE_PUBLIC_KEY", - "LANGFUSE_SECRET_KEY", - "LANGFUSE_HOST", - ] - _langfuse_env_vars = {} - for _var in _langfuse_vars: - env_variable = environment_variables.get(_var, None) - if env_variable is None: - _langfuse_env_vars[_var] = None - else: - # decode + decrypt the value - decrypted_value = decrypt_value_helper( - value=env_variable, key=_var - ) - _langfuse_env_vars[_var] = decrypted_value - - _data_to_return.append( - {"name": _callback, "variables": _langfuse_env_vars} - ) + _data_to_return.append(process_callback(_callback, "success")) + + for _callback in _failure_callbacks: + _data_to_return.append(process_callback(_callback, "failure")) + + for _callback in _generic_callbacks: + _data_to_return.append(process_callback(_callback, "generic")) # Check if slack alerting is on _alerting = _general_settings.get("alerting", []) diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index 1f4bf806c1..3aeb8a840c 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -2371,3 +2371,166 @@ def test_non_root_ui_path_logic(monkeypatch, tmp_path, ui_exists, ui_has_content error_calls = [call[0][0] for call in mock_logger.error.call_args_list] assert any("Path exists:" in call for call in error_calls) assert mock_logger.info.call_count == 0 + + +@pytest.mark.asyncio +async def test_get_config_callbacks_with_all_types(client_no_auth): + """ + Test that /get/config/callbacks returns all three callback types: + - success_callback with type="success" + - failure_callback with type="failure" + - callbacks (generic) with type="generic" + """ + from litellm.proxy.proxy_server import ProxyConfig + + # Create a mock config with all three callback types + mock_config_data = { + "litellm_settings": { + "success_callback": ["langfuse", "braintrust"], + "failure_callback": ["sentry"], + "callbacks": ["otel", "langsmith"] + }, + "environment_variables": { + "LANGFUSE_PUBLIC_KEY": "test-public-key", + "LANGFUSE_SECRET_KEY": "test-secret-key", + "LANGFUSE_HOST": "https://test.langfuse.com", + "BRAINTRUST_API_KEY": "test-braintrust-key", + "OTEL_EXPORTER": "otlp", + "OTEL_ENDPOINT": "http://localhost:4317", + "LANGSMITH_API_KEY": "test-langsmith-key", + }, + "general_settings": {} + } + + proxy_config = getattr(litellm.proxy.proxy_server, "proxy_config") + + with patch.object( + proxy_config, "get_config", new=AsyncMock(return_value=mock_config_data) + ), patch( + "litellm.proxy.proxy_server.decrypt_value_helper", + side_effect=lambda value, key=None: value + ): + response = client_no_auth.get("/get/config/callbacks") + + assert response.status_code == 200 + result = response.json() + + # Verify response structure + assert "status" in result + assert result["status"] == "success" + assert "callbacks" in result + + callbacks = result["callbacks"] + + # Verify we have all 5 callbacks (2 success + 1 failure + 2 generic) + assert len(callbacks) == 5 + + # Group callbacks by type + success_callbacks = [cb for cb in callbacks if cb.get("type") == "success"] + failure_callbacks = [cb for cb in callbacks if cb.get("type") == "failure"] + generic_callbacks = [cb for cb in callbacks if cb.get("type") == "generic"] + + # Verify all callbacks have required fields + for callback in callbacks: + assert "name" in callback + assert "variables" in callback + assert "type" in callback + assert callback["type"] in ["success", "failure", "generic"] + + # Verify success callbacks + assert len(success_callbacks) == 2 + success_names = [cb["name"] for cb in success_callbacks] + assert "langfuse" in success_names + assert "braintrust" in success_names + + # Verify failure callbacks + assert len(failure_callbacks) == 1 + assert failure_callbacks[0]["name"] == "sentry" + + # Verify generic callbacks + assert len(generic_callbacks) == 2 + generic_names = [cb["name"] for cb in generic_callbacks] + assert "otel" in generic_names + assert "langsmith" in generic_names + + +@pytest.mark.asyncio +async def test_get_config_callbacks_environment_variables(client_no_auth): + """ + Test that /get/config/callbacks correctly includes environment variables + for each callback type with proper decryption. + """ + from litellm.proxy.proxy_server import ProxyConfig + + # Create a mock config with callbacks and their env vars + mock_config_data = { + "litellm_settings": { + "success_callback": ["langfuse"], + "failure_callback": [], + "callbacks": ["otel"] + }, + "environment_variables": { + "LANGFUSE_PUBLIC_KEY": "encrypted-public-key", + "LANGFUSE_SECRET_KEY": "encrypted-secret-key", + "LANGFUSE_HOST": "https://cloud.langfuse.com", + "OTEL_EXPORTER": "otlp", + "OTEL_ENDPOINT": "http://localhost:4317", + "OTEL_HEADERS": "key=value", + }, + "general_settings": {} + } + + # Mock decrypt to prepend "decrypted-" to values + def mock_decrypt(value, key=None): + if value and isinstance(value, str) and "encrypted" in value: + return f"decrypted-{value}" + return value + + proxy_config = getattr(litellm.proxy.proxy_server, "proxy_config") + + with patch.object( + proxy_config, "get_config", new=AsyncMock(return_value=mock_config_data) + ), patch( + "litellm.proxy.proxy_server.decrypt_value_helper", + side_effect=mock_decrypt + ): + response = client_no_auth.get("/get/config/callbacks") + + assert response.status_code == 200 + result = response.json() + + callbacks = result["callbacks"] + + # Find langfuse callback (success type) + langfuse_callback = next( + (cb for cb in callbacks if cb["name"] == "langfuse"), None + ) + assert langfuse_callback is not None + assert langfuse_callback["type"] == "success" + assert "variables" in langfuse_callback + + # Verify langfuse env vars are present and decrypted + langfuse_vars = langfuse_callback["variables"] + assert "LANGFUSE_PUBLIC_KEY" in langfuse_vars + assert langfuse_vars["LANGFUSE_PUBLIC_KEY"] == "decrypted-encrypted-public-key" + assert "LANGFUSE_SECRET_KEY" in langfuse_vars + assert langfuse_vars["LANGFUSE_SECRET_KEY"] == "decrypted-encrypted-secret-key" + assert "LANGFUSE_HOST" in langfuse_vars + assert langfuse_vars["LANGFUSE_HOST"] == "https://cloud.langfuse.com" + + # Find otel callback (generic type) + otel_callback = next( + (cb for cb in callbacks if cb["name"] == "otel"), None + ) + assert otel_callback is not None + assert otel_callback["type"] == "generic" + assert "variables" in otel_callback + + # Verify otel env vars are present + otel_vars = otel_callback["variables"] + assert "OTEL_EXPORTER" in otel_vars + assert otel_vars["OTEL_EXPORTER"] == "otlp" + assert "OTEL_ENDPOINT" in otel_vars + assert otel_vars["OTEL_ENDPOINT"] == "http://localhost:4317" + assert "OTEL_HEADERS" in otel_vars + assert otel_vars["OTEL_HEADERS"] == "key=value" diff --git a/ui/litellm-dashboard/src/components/settings.test.tsx b/ui/litellm-dashboard/src/components/settings.test.tsx new file mode 100644 index 0000000000..72a265eadd --- /dev/null +++ b/ui/litellm-dashboard/src/components/settings.test.tsx @@ -0,0 +1,136 @@ +import { render, screen, fireEvent, waitFor } from "@testing-library/react"; +import { describe, expect, it, beforeAll, beforeEach, vi } from "vitest"; +import Settings from "./settings"; +import * as networking from "./networking"; + +beforeAll(() => { + Object.defineProperty(window, "matchMedia", { + writable: true, + value: (query: string) => ({ + matches: false, + media: query, + onchange: null, + addListener: () => {}, + removeListener: () => {}, + addEventListener: () => {}, + removeEventListener: () => {}, + dispatchEvent: () => true, + }), + }); +}); + +const mockCallbacksData = { + callbacks: [ + { + name: "langfuse", + type: "success", + variables: { + LANGFUSE_PUBLIC_KEY: "test_key", + LANGFUSE_SECRET_KEY: "test_secret", + }, + }, + { + name: "datadog", + type: "success", + variables: { + DD_API_KEY: "test_dd_key", + }, + }, + ], + available_callbacks: [ + { + litellm_callback_name: "langfuse", + ui_callback_name: "Langfuse", + litellm_callback_params: ["LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY"], + }, + { + litellm_callback_name: "datadog", + ui_callback_name: "Datadog", + litellm_callback_params: ["DD_API_KEY"], + }, + ], + alerts: [], +}; + +describe("Settings", () => { + it("should render the settings page", () => { + render(); + }); +}); + +describe("Logging Callbacks Section", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should display the list of active callbacks", async () => { + vi.spyOn(networking, "getCallbacksCall").mockResolvedValue(mockCallbacksData); + + render(); + + await waitFor( + () => { + expect(screen.getByText("langfuse")).toBeInTheDocument(); + expect(screen.getByText("datadog")).toBeInTheDocument(); + }, + { timeout: 3000 }, + ); + }); + + it("should open add callback modal and display form", async () => { + vi.spyOn(networking, "getCallbacksCall").mockResolvedValue(mockCallbacksData); + + render(); + + await waitFor(() => { + expect(screen.getByText("langfuse")).toBeInTheDocument(); + }); + + const addButton = screen.getByText("Add Callback"); + fireEvent.click(addButton); + + await waitFor(() => { + expect(screen.getByText("Add Logging Callback")).toBeInTheDocument(); + expect(screen.getByText("LiteLLM Docs: Logging")).toBeInTheDocument(); + }); + }); + + it("should successfully delete a callback", async () => { + const getCallbacksSpy = vi.spyOn(networking, "getCallbacksCall").mockResolvedValue(mockCallbacksData); + const deleteCallbackSpy = vi.spyOn(networking, "deleteCallback").mockResolvedValue(undefined); + + const { container } = render( + , + ); + + await waitFor(() => { + expect(screen.getByText("langfuse")).toBeInTheDocument(); + }); + + const trashIcons = container.querySelectorAll("svg"); + const trashIcon = Array.from(trashIcons).find((svg) => { + const parentElement = svg.parentElement; + return parentElement?.className.includes("text-red") || parentElement?.outerHTML.includes("red"); + }); + + expect(trashIcon).toBeDefined(); + if (trashIcon && trashIcon.parentElement) { + fireEvent.click(trashIcon.parentElement); + } + + await waitFor(() => { + const modalText = screen.getByText((content, element) => { + return element?.tagName.toLowerCase() === "p" && content.includes("Are you sure you want to delete"); + }); + expect(modalText).toBeInTheDocument(); + }); + + const deleteButton = screen.getByRole("button", { name: "Delete" }); + fireEvent.click(deleteButton); + + await waitFor(() => { + expect(deleteCallbackSpy).toHaveBeenCalledWith("test-token", "langfuse"); + expect(getCallbacksSpy).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 727c8b9b51..a275214918 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -19,11 +19,12 @@ import { Tab, SelectItem, Icon, + Badge, } from "@tremor/react"; import { PencilAltIcon, TrashIcon } from "@heroicons/react/outline"; -import { Modal, Typography, Form, Input, Select, Button as Button2 } from "antd"; +import { Modal, Typography, Form, Input, Select, Button as Button2, Tooltip } from "antd"; import NotificationsManager from "./molecules/notifications_manager"; import EmailSettings from "./email_settings"; @@ -32,10 +33,7 @@ const { Title, Paragraph } = Typography; import { getCallbacksCall, setCallbacksCall, serviceHealthCheck, deleteCallback } from "./networking"; import AlertingSettings from "./alerting/alerting_settings"; import FormItem from "antd/es/form/FormItem"; -import { - CALLBACK_CONFIGS, - getCallbackById, -} from "./callback_info_helpers"; +import { CALLBACK_CONFIGS, getCallbackById } from "./callback_info_helpers"; import { parseErrorMessage } from "./shared/errorUtils"; interface SettingsPageProps { accessToken: string | null; @@ -60,6 +58,7 @@ interface AlertingVariables { interface AlertingObject { name: string; + type?: "success" | "failure" | "generic"; variables: AlertingVariables; } @@ -216,10 +215,10 @@ const Settings: React.FC = ({ accessToken, userRole, userID, const handleSelectedCallbackChange = (callbackName: string) => { setSelectedCallback(callbackName); - + // Get the callback configuration using the new clean structure const callbackConfig = getCallbackById(callbackName); - + // Get the parameters from the callback configuration if (callbackConfig?.dynamic_params) { const params = Object.keys(callbackConfig.dynamic_params); @@ -228,7 +227,7 @@ const Settings: React.FC = ({ accessToken, userRole, userID, setSelectedCallbackParams([]); } }; - + const handleSaveAlerts = async () => { if (!accessToken) { return; @@ -416,54 +415,94 @@ const Settings: React.FC = ({ accessToken, userRole, userID, Active Logging Callbacks - + Callback Name - {/* Callback Env Vars */} + Callback Type + Actions - {callbacks.map((callback, index) => ( - - - {callback.name} - - - - { - setSelectedEditCallback(callback); - setShowEditCallback(true); - }} - /> - handleDeleteCallback(callback.name)} - className="text-red-500 hover:text-red-700 cursor-pointer" - /> - - - - - ))} + {callbacks.map((callback, index) => { + const canEdit = !callback.type || callback.type === "success"; + const tooltipMessage = + callback.type === "failure" + ? "Modifications and deletion of failure type callbacks are not yet supported in the UI" + : callback.type === "generic" + ? "Modifications and deletion of generic type callbacks are not yet supported in the UI" + : ""; + + const getBadgeColor = (type?: string) => { + if (type === "success") return "green"; + if (type === "failure") return "red"; + if (type === "generic") return "blue"; + return "gray"; + }; + + return ( + + + {callback.name} + + + {callback.type ? ( + {callback.type} + ) : ( + success + )} + + +
+ + { + if (canEdit) { + setSelectedEditCallback(callback); + setShowEditCallback(true); + } + }} + className={canEdit ? "cursor-pointer" : "opacity-40 cursor-not-allowed"} + /> + + + { + if (canEdit) { + handleDeleteCallback(callback.name); + } + }} + className={ + canEdit + ? "text-red-500 hover:text-red-700 cursor-pointer" + : "text-red-300 opacity-40 cursor-not-allowed" + } + /> + + +
+ + + ); + })}
@@ -594,124 +633,109 @@ const Settings: React.FC = ({ accessToken, userRole, userID, wrapperCol={{ span: 16 }} labelAlign="left" > - + - (option?.children?.toString() ?? "") - .toLowerCase() - .includes(input.toLowerCase()) - } - onChange={(value) => { - handleSelectedCallbackChange(value); - }} - > - {CALLBACK_CONFIGS.map((callbackConfig) => ( - -
-
- {/* eslint-disable-next-line @next/next/no-img-element */} - {`${callbackConfig.displayName} { - e.currentTarget.style.display = 'none'; - }} - /> -
- - {callbackConfig.displayName} - + {CALLBACK_CONFIGS.map((callbackConfig) => ( + +
+
+ {/* eslint-disable-next-line @next/next/no-img-element */} + {`${callbackConfig.displayName} { + e.currentTarget.style.display = "none"; + }} + />
- - ))} - - + {callbackConfig.displayName} +
+ + ))} + + - {selectedCallbackParams && selectedCallbackParams.length > 0 && ( -
- {selectedCallbackParams.map((param) => { - // Get the callback configuration to look up parameter types - const callbackConfig = getCallbackById(selectedCallback || ''); - const paramType = callbackConfig?.dynamic_params[param] || "text"; - - const fieldLabel = param.replace(/_/g, " ").replace(/\b\w/g, l => l.toUpperCase()); - - return ( - - {fieldLabel} - * - - } - name={param} - key={param} - className="mb-4" - rules={[ - { - required: true, - message: `Please enter the ${fieldLabel.toLowerCase()}`, - }, - ]} - > - {paramType === "password" ? ( - - ) : paramType === "number" ? ( - - ) : ( - - )} - - ); - })} -
- )} + {selectedCallbackParams && selectedCallbackParams.length > 0 && ( +
+ {selectedCallbackParams.map((param) => { + // Get the callback configuration to look up parameter types + const callbackConfig = getCallbackById(selectedCallback || ""); + const paramType = callbackConfig?.dynamic_params[param] || "text"; -
- - - Add Callback - + const fieldLabel = param.replace(/_/g, " ").replace(/\b\w/g, (l) => l.toUpperCase()); + + return ( + + {fieldLabel} + * + + } + name={param} + key={param} + className="mb-4" + rules={[ + { + required: true, + message: `Please enter the ${fieldLabel.toLowerCase()}`, + }, + ]} + > + {paramType === "password" ? ( + + ) : paramType === "number" ? ( + + ) : ( + + )} + + ); + })}
+ )} + +
+ + Add Callback +