ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(UI) Error Logs improvements - Store Raw proxy server request for success and failure (#8917)

Browse Source 
* ui render error logs

* add proxy_server_request to spend logs metadata

* _add_proxy_server_request_to_metadata

* proxy track cost callback fixes

* _add_proxy_server_request_to_metadata

* test spend logs

* test_async_post_call_failure_hook
This commit is contained in:
Ishaan Jaff 2025-03-01 16:26:47 -08:00 committed by GitHub
parent bc9b3e4847
commit 1008da7c41
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 72 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
litellm/proxy/_types.py
View File

@ -1858,6 +1858,7 @@ class SpendLogsMetadata(TypedDict):
requester_ip_address: Optional[str]
applied_guardrails: Optional[List[str]]
status: StandardLoggingPayloadStatus
proxy_server_request: Optional[str]
error_information: Optional[StandardLoggingPayloadErrorInformation]

11
litellm/proxy/hooks/proxy_track_cost_callback.py
View File

@ -56,12 +56,13 @@ class _ProxyDBLogger(CustomLogger):
existing_metadata: dict = request_data.get("metadata", None) or {}
existing_metadata.update(_metadata)
existing_metadata["proxy_server_request"] = (
request_data.get("proxy_server_request", {}) or {}
)
request_data["litellm_params"] = {}
request_data["litellm_params"]["metadata"] = existing_metadata
if "litellm_params" not in request_data:
request_data["litellm_params"] = {}
request_data["litellm_params"]["proxy_server_request"] = (
request_data.get("proxy_server_request") or {}
)
request_data["litellm_params"]["metadata"] = existing_metadata
await update_database(
token=user_api_key_dict.api_key,
response_cost=0.0,

29
litellm/proxy/spend_tracking/spend_tracking_utils.py
View File

@ -49,6 +49,7 @@ def _get_spend_logs_metadata(
applied_guardrails=None,
status=None or "success",
error_information=None,
proxy_server_request=None,
)
verbose_proxy_logger.debug(
"getting payload for SpendLogs, available keys in metadata: "
@ -84,6 +85,9 @@ def get_logging_payload( # noqa: PLR0915
metadata = (
litellm_params.get("metadata", {}) or {}
) # if litellm_params['metadata'] == None
metadata = _add_proxy_server_request_to_metadata(
metadata=metadata, litellm_params=litellm_params
)
completion_start_time = kwargs.get("completion_start_time", end_time)
call_type = kwargs.get("call_type")
cache_hit = kwargs.get("cache_hit", False)
@ -299,19 +303,24 @@ def _get_messages_for_spend_logs_payload(
standard_logging_payload: Optional[StandardLoggingPayload],
metadata: Optional[dict] = None,
) -> str:
if _should_store_prompts_and_responses_in_spend_logs():
metadata = metadata or {}
if metadata.get("status", None) == "failure":
_proxy_server_request = metadata.get("proxy_server_request", {})
_request_body = _proxy_server_request.get("body", {}) or {}
return json.dumps(_request_body, default=str)
else:
if standard_logging_payload is None:
return "{}"
return json.dumps(standard_logging_payload.get("messages", {}))
return "{}"
def _add_proxy_server_request_to_metadata(
metadata: dict,
litellm_params: dict,
) -> dict:
"""
Only store if _should_store_prompts_and_responses_in_spend_logs() is True
"""
if _should_store_prompts_and_responses_in_spend_logs():
_proxy_server_request = litellm_params.get("proxy_server_request", {})
_request_body = _proxy_server_request.get("body", {}) or {}
_request_body_json_str = json.dumps(_request_body, default=str)
metadata["proxy_server_request"] = _request_body_json_str
return metadata
def _get_response_for_spend_logs_payload(
payload: Optional[StandardLoggingPayload],
) -> str:

6
tests/litellm/proxy/hooks/test_proxy_track_cost_callback.py
View File

@ -61,6 +61,7 @@ async def test_async_post_call_failure_hook():
# Check the arguments passed to update_database
call_args = mock_update_database.call_args[1]
print("call_args", json.dumps(call_args, indent=4, default=str))
assert call_args["token"] == "test_api_key"
assert call_args["response_cost"] == 0.0
assert call_args["user_id"] == "test_user_id"
@ -71,10 +72,11 @@ async def test_async_post_call_failure_hook():
# Check that metadata was properly updated
assert "litellm_params" in call_args["kwargs"]
assert call_args["kwargs"]["litellm_params"]["proxy_server_request"] == {
"request_id": "test_request_id"
}
metadata = call_args["kwargs"]["litellm_params"]["metadata"]
assert metadata["user_api_key"] == "test_api_key"
assert metadata["status"] == "failure"
assert "error_information" in metadata
assert metadata["original_key"] == "original_value"
assert "proxy_server_request" in metadata
assert metadata["proxy_server_request"]["request_id"] == "test_request_id"

15
tests/logging_callback_tests/test_spend_logs.py
View File

@ -98,6 +98,7 @@ def test_spend_logs_payload(model_id: Optional[str]):
"caching_groups": None,
"error_information": None,
"status": "success",
"proxy_server_request": "{}",
"raw_request": "\n\nPOST Request Sent from LiteLLM:\ncurl -X POST \\\nhttps://openai-gpt-4-test-v-1.openai.azure.com//openai/ \\\n-H 'Authorization: *****' \\\n-d '{'model': 'chatgpt-v-2', 'messages': [{'role': 'system', 'content': 'you are a helpful assistant.\\n'}, {'role': 'user', 'content': 'bom dia'}], 'stream': False, 'max_tokens': 10, 'user': '116544810872468347480', 'extra_body': {}}'\n",
},
"model_info": {
@ -357,17 +358,29 @@ def test_spend_logs_payload_with_prompts_enabled(monkeypatch):
},
"request_tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"],
}
litellm_params = {
"proxy_server_request": {
"body": {
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello!"}],
}
}
}
input_args["kwargs"]["standard_logging_object"] = standard_logging_payload
input_args["kwargs"]["litellm_params"] = litellm_params
payload: SpendLogsPayload = get_logging_payload(**input_args)
print("json payload: ", json.dumps(payload, indent=4, default=str))
# Verify messages and response are included in payload
assert payload["messages"] == json.dumps([{"role": "user", "content": "Hello!"}])
assert payload["response"] == json.dumps(
{"role": "assistant", "content": "Hi there!"}
)
parsed_metadata = json.loads(payload["metadata"])
assert parsed_metadata["proxy_server_request"] == json.dumps(
{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello!"}]}
)
# Clean up - reset general_settings
general_settings["store_prompts_in_spend_logs"] = False

32
ui/litellm-dashboard/src/components/view_logs/index.tsx
View File

@ -566,6 +566,15 @@ export default function SpendLogsTable({
}
function RequestViewer({ row }: { row: Row<LogEntry> }) {
// Helper function to clean metadata by removing specific fields
const getCleanedMetadata = (metadata: any) => {
const cleanedMetadata = {...metadata};
if ('proxy_server_request' in cleanedMetadata) {
delete cleanedMetadata.proxy_server_request;
}
return cleanedMetadata;
};
const formatData = (input: any) => {
if (typeof input === "string") {
try {
@ -577,6 +586,16 @@ function RequestViewer({ row }: { row: Row<LogEntry> }) {
return input;
};
// New helper function to get raw request
const getRawRequest = () => {
// First check if proxy_server_request exists in metadata
if (row.original.metadata?.proxy_server_request) {
return formatData(row.original.metadata.proxy_server_request);
}
// Fall back to messages if proxy_server_request is empty
return formatData(row.original.messages);
};
// Extract error information from metadata if available
const hasError = row.original.metadata?.status === "failure";
const errorInfo = hasError ? row.original.metadata?.error_information : null;
@ -666,7 +685,7 @@ function RequestViewer({ row }: { row: Row<LogEntry> }) {
<div className="flex justify-between items-center p-4 border-b">
<h3 className="text-lg font-medium">Request</h3>
<button
onClick={() => navigator.clipboard.writeText(JSON.stringify(formatData(row.original.messages), null, 2))}
onClick={() => navigator.clipboard.writeText(JSON.stringify(getRawRequest(), null, 2))}
className="p-1 hover:bg-gray-200 rounded"
title="Copy request"
>
@ -677,7 +696,7 @@ function RequestViewer({ row }: { row: Row<LogEntry> }) {
</button>
</div>
<div className="p-4 overflow-auto max-h-96">
<pre className="text-xs font-mono whitespace-pre-wrap break-all">{JSON.stringify(formatData(row.original.messages), null, 2)}</pre>
<pre className="text-xs font-mono whitespace-pre-wrap break-all">{JSON.stringify(getRawRequest(), null, 2)}</pre>
</div>
</div>
@ -738,7 +757,10 @@ function RequestViewer({ row }: { row: Row<LogEntry> }) {
<div className="flex justify-between items-center p-4 border-b">
<h3 className="text-lg font-medium">Metadata</h3>
<button
onClick={() => navigator.clipboard.writeText(JSON.stringify(row.original.metadata, null, 2))}
onClick={() => {
const cleanedMetadata = getCleanedMetadata(row.original.metadata);
navigator.clipboard.writeText(JSON.stringify(cleanedMetadata, null, 2));
}}
className="p-1 hover:bg-gray-200 rounded"
title="Copy metadata"
>
@ -749,7 +771,9 @@ function RequestViewer({ row }: { row: Row<LogEntry> }) {
</button>
</div>
<div className="p-4 overflow-auto max-h-64">
<pre className="text-xs font-mono whitespace-pre-wrap break-all">{JSON.stringify(row.original.metadata, null, 2)}</pre>
<pre className="text-xs font-mono whitespace-pre-wrap break-all">
{JSON.stringify(getCleanedMetadata(row.original.metadata), null, 2)}
</pre>
</div>
</div>
)}