From 41566722af6a198ee2c2a11c99ac1acf7d258554 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 21 Nov 2025 08:53:18 -0800 Subject: [PATCH] [Feat] UI - Prompt Management - Allow testing prompts with Chat UI (#16898) * TestPromptRequest * add prompts/test endpoint for testing prompt * TestPromptTestEndpoint * feat: working v1 of this ui * workig prompt endpoints * add chat ui for prompts * add conversation panel * add init chat ui --- litellm/proxy/prompts/prompt_endpoints.py | 159 +++++++++++- litellm/types/proxy/prompt_endpoints.py | 10 + .../test_prompt_test_endpoint.py | 134 ++++++++++ .../prompt_editor_view/ConversationPanel.tsx | 21 -- .../conversation_panel/EmptyState.tsx | 22 ++ .../conversation_panel/MessageBubble.tsx | 115 ++++++++ .../conversation_panel/MessageInput.tsx | 71 +++++ .../conversation_panel/MessageList.tsx | 42 +++ .../conversation_panel/VariableInput.tsx | 44 ++++ .../conversation_panel/VariableWarning.tsx | 38 +++ .../conversation_panel/index.tsx | 78 ++++++ .../conversation_panel/types.ts | 16 ++ .../conversation_panel/useConversation.ts | 245 ++++++++++++++++++ .../prompts/prompt_editor_view/index.tsx | 8 +- 14 files changed, 978 insertions(+), 25 deletions(-) create mode 100644 litellm/types/proxy/prompt_endpoints.py create mode 100644 tests/proxy_unit_tests/test_prompt_test_endpoint.py delete mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/ConversationPanel.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/EmptyState.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageBubble.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageInput.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageList.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/VariableInput.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/VariableWarning.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/index.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/types.ts create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/useConversation.ts diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index b4677ed374..373f993d3e 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -6,7 +6,15 @@ import tempfile from pathlib import Path from typing import Any, Dict, List, Optional, cast -from fastapi import APIRouter, Depends, File, HTTPException, UploadFile +from fastapi import ( + APIRouter, + Depends, + File, + HTTPException, + Request, + Response, + UploadFile, +) from pydantic import BaseModel from litellm._logging import verbose_proxy_logger @@ -20,6 +28,7 @@ from litellm.types.prompts.init_prompts import ( PromptSpec, PromptTemplateBase, ) +from litellm.types.proxy.prompt_endpoints import TestPromptRequest router = APIRouter() @@ -667,6 +676,154 @@ async def patch_prompt( raise HTTPException(status_code=500, detail=str(e)) +@router.post( + "/prompts/test", + tags=["Prompt Management"], + dependencies=[Depends(user_api_key_auth)], +) +async def test_prompt( + request: TestPromptRequest, + fastapi_request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Test a prompt by rendering it with variables and executing an LLM call. + + This endpoint allows testing prompts before saving them to the database. + The response is always streamed. + + 👉 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management) + + Example Request: + ```bash + curl -X POST "http://localhost:4000/prompts/test" \\ + -H "Authorization: Bearer " \\ + -H "Content-Type: application/json" \\ + -d '{ + "dotprompt_content": "---\\nmodel: gpt-4o\\ntemperature: 0.7\\n---\\n\\nUser: Hello {{name}}", + "prompt_variables": { + "name": "World" + } + }' + ``` + """ + from pydantic import BaseModel + + from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager + from litellm.integrations.dotprompt.prompt_manager import ( + PromptManager, + PromptTemplate, + ) + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + try: + # Parse the dotprompt content and create PromptTemplate + prompt_manager = PromptManager() + frontmatter, template_content = prompt_manager._parse_frontmatter( + content=request.dotprompt_content + ) + + # Create PromptTemplate to leverage existing parameter extraction logic + template = PromptTemplate( + content=template_content, + metadata=frontmatter, + template_id="test_prompt" + ) + + # Extract model from template + if not template.model: + raise HTTPException( + status_code=400, + detail="Model is required in dotprompt metadata" + ) + + # Always render the template to extract system messages and other metadata + variables = request.prompt_variables or {} + rendered_content = prompt_manager.jinja_env.from_string( + template_content + ).render(**variables) + + # Convert rendered content to messages using DotpromptManager's method + dotprompt_manager = DotpromptManager() + rendered_messages = dotprompt_manager._convert_to_messages( + rendered_content=rendered_content + ) + + if not rendered_messages: + raise HTTPException( + status_code=400, + detail="No messages found in rendered prompt" + ) + + # If conversation history is provided, use it but preserve system messages + if request.conversation_history: + # Extract system messages from rendered prompt + system_messages = [msg for msg in rendered_messages if msg.get("role") == "system"] + # Use conversation history for user/assistant messages + messages = system_messages + request.conversation_history + else: + messages = rendered_messages + + # Use PromptTemplate's optional_params which already extracts all parameters + optional_params = template.optional_params.copy() + + # Always stream the response + optional_params["stream"] = True + + # Build request data for chat completion + data = { + "model": template.model, + "messages": messages, + } + data.update(optional_params) + + # Use ProxyBaseLLMRequestProcessing to go through all proxy logic + base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + result = await base_llm_response_processor.base_process_llm_request( + request=fastapi_request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="acompletion", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + + if isinstance(result, BaseModel): + return result.model_dump(exclude_none=True, exclude_unset=True) + else: + return result + + except HTTPException as e: + raise e + except Exception as e: + verbose_proxy_logger.exception(f"Error testing prompt: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.post( "/utils/dotprompt_json_converter", tags=["prompts", "utils"], diff --git a/litellm/types/proxy/prompt_endpoints.py b/litellm/types/proxy/prompt_endpoints.py new file mode 100644 index 0000000000..620a565b0a --- /dev/null +++ b/litellm/types/proxy/prompt_endpoints.py @@ -0,0 +1,10 @@ +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel + + +class TestPromptRequest(BaseModel): + dotprompt_content: str + prompt_variables: Optional[Dict[str, Any]] = None + conversation_history: Optional[List[Dict[str, str]]] = None + diff --git a/tests/proxy_unit_tests/test_prompt_test_endpoint.py b/tests/proxy_unit_tests/test_prompt_test_endpoint.py new file mode 100644 index 0000000000..327f60e3d7 --- /dev/null +++ b/tests/proxy_unit_tests/test_prompt_test_endpoint.py @@ -0,0 +1,134 @@ +""" +Test /prompts/test endpoint for testing prompts before saving +""" + +import pytest +from unittest.mock import AsyncMock, MagicMock, patch +from fastapi import HTTPException + + +class TestPromptTestEndpoint: + """ + Tests the /prompts/test endpoint that allows testing prompts with variables + """ + + @pytest.mark.asyncio + async def test_parse_dotprompt_with_variables(self): + """ + Test that dotprompt content is parsed and variables are rendered correctly + """ + from litellm.integrations.dotprompt.prompt_manager import PromptManager + + dotprompt_content = """--- +model: gpt-4o +temperature: 0.7 +max_tokens: 100 +--- + +User: Hello {{name}}, how are you?""" + + # Parse the dotprompt + prompt_manager = PromptManager() + frontmatter, template_content = prompt_manager._parse_frontmatter( + content=dotprompt_content + ) + + assert frontmatter["model"] == "gpt-4o" + assert frontmatter["temperature"] == 0.7 + assert frontmatter["max_tokens"] == 100 + assert "{{name}}" in template_content + + # Render with variables + from jinja2 import Environment + + jinja_env = Environment( + variable_start_string="{{", + variable_end_string="}}", + ) + jinja_template = jinja_env.from_string(template_content) + rendered = jinja_template.render(name="World") + + assert "Hello World" in rendered + assert "{{name}}" not in rendered + + @pytest.mark.asyncio + async def test_convert_to_messages_format(self): + """ + Test that rendered prompt is converted to OpenAI messages format + """ + import re + + rendered_content = """System: You are a helpful assistant. + +User: Hello World, how are you?""" + + messages = [] + role_pattern = r"^(System|User|Assistant|Developer):\s*(.*?)(?=\n(?:System|User|Assistant|Developer):|$)" + matches = list( + re.finditer( + pattern=role_pattern, + string=rendered_content.strip(), + flags=re.MULTILINE | re.DOTALL, + ) + ) + + for match in matches: + role = match.group(1).lower() + content = match.group(2).strip() + + if role == "developer": + role = "system" + + if content: + messages.append({"role": role, "content": content}) + + assert len(messages) == 2 + assert messages[0]["role"] == "system" + assert "helpful assistant" in messages[0]["content"] + assert messages[1]["role"] == "user" + assert "Hello World" in messages[1]["content"] + + @pytest.mark.asyncio + async def test_single_message_without_role(self): + """ + Test that content without role markers is treated as a user message + """ + import re + + rendered_content = "Just a plain message without any role markers" + + messages = [] + role_pattern = r"^(System|User|Assistant|Developer):\s*(.*?)(?=\n(?:System|User|Assistant|Developer):|$)" + matches = list( + re.finditer( + pattern=role_pattern, + string=rendered_content.strip(), + flags=re.MULTILINE | re.DOTALL, + ) + ) + + if not matches: + messages.append({"role": "user", "content": rendered_content.strip()}) + + assert len(messages) == 1 + assert messages[0]["role"] == "user" + assert messages[0]["content"] == rendered_content + + @pytest.mark.asyncio + async def test_missing_model_raises_error(self): + """ + Test that missing model in frontmatter raises an error + """ + from litellm.integrations.dotprompt.prompt_manager import PromptManager + + dotprompt_content = """--- +temperature: 0.7 +--- + +User: Hello""" + + prompt_manager = PromptManager() + frontmatter, _ = prompt_manager._parse_frontmatter(content=dotprompt_content) + + model = frontmatter.get("model") + assert model is None diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/ConversationPanel.tsx b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/ConversationPanel.tsx deleted file mode 100644 index a827349d6e..0000000000 --- a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/ConversationPanel.tsx +++ /dev/null @@ -1,21 +0,0 @@ -import React from "react"; -import { MessageSquareIcon } from "lucide-react"; - -const ConversationPanel: React.FC = () => { - return ( -
-
-
-
- -
-

Your conversation will appear here

-

Save the prompt to test it

-
-
-
- ); -}; - -export default ConversationPanel; - diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/EmptyState.tsx b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/EmptyState.tsx new file mode 100644 index 0000000000..57a6a54760 --- /dev/null +++ b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/EmptyState.tsx @@ -0,0 +1,22 @@ +import React from "react"; +import { RobotOutlined } from "@ant-design/icons"; + +interface EmptyStateProps { + hasVariables: boolean; +} + +const EmptyState: React.FC = ({ hasVariables }) => { + return ( +
+ + + {hasVariables + ? "Fill in the variables above, then type a message to start testing" + : "Type a message below to start testing your prompt"} + +
+ ); +}; + +export default EmptyState; + diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageBubble.tsx b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageBubble.tsx new file mode 100644 index 0000000000..e5e4f46b40 --- /dev/null +++ b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageBubble.tsx @@ -0,0 +1,115 @@ +import React from "react"; +import { RobotOutlined, UserOutlined } from "@ant-design/icons"; +import ReactMarkdown from "react-markdown"; +import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; +import { coy } from "react-syntax-highlighter/dist/esm/styles/prism"; +import ResponseMetrics from "../../../playground/chat_ui/ResponseMetrics"; +import { Message } from "./types"; + +interface MessageBubbleProps { + message: Message; +} + +const MessageBubble: React.FC = ({ message }) => { + return ( +
+
+
+
+ {message.role === "user" ? ( + + ) : ( + + )} +
+ {message.role} + {message.role === "assistant" && message.model && ( + + {message.model} + + )} +
+ +
+ {message.role === "assistant" ? ( + & { + inline?: boolean; + node?: any; + }) { + const match = /language-(\w+)/.exec(className || ""); + return !inline && match ? ( + + {String(children).replace(/\n$/, "")} + + ) : ( + + {children} + + ); + }, + pre: ({ node, ...props }) => ( + 
+                ),
+              }}
+            >
+              {message.content}
+            
+          ) : (
+            
{message.content}

+          )}
+
+          {message.role === "assistant" &&
+            (message.timeToFirstToken || message.totalLatency || message.usage) && (
+              
+            )}
+
+
+
+ ); +}; + +export default MessageBubble; + diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageInput.tsx b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageInput.tsx new file mode 100644 index 0000000000..9ec232dee4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageInput.tsx @@ -0,0 +1,71 @@ +import React from "react"; +import { ArrowUpOutlined } from "@ant-design/icons"; +import { Button as TremorButton } from "@tremor/react"; +import { Input } from "antd"; + +const { TextArea } = Input; + +interface MessageInputProps { + inputMessage: string; + isLoading: boolean; + isDisabled: boolean; + onInputChange: (value: string) => void; + onSend: () => void; + onKeyDown: (event: React.KeyboardEvent) => void; + onCancel: () => void; +} + +const MessageInput: React.FC = ({ + inputMessage, + isLoading, + isDisabled, + onInputChange, + onSend, + onKeyDown, + onCancel, +}) => { + return ( +
+
+