ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(prometheus): emit per-token-type detail metrics (LIT-3220) (#28372) (#28378)

Browse Source 
* feat(prometheus): emit per-token-type detail metrics (LIT-3220) (#28372)

Adds five sparse counter metrics that break out the token detail
fields providers already report in `usage.prompt_tokens_details` and
`usage.completion_tokens_details`:

  - litellm_input_cached_tokens_metric            (provider prompt-cache reads)
  - litellm_input_cache_creation_tokens_metric    (Anthropic prompt-cache writes)
  - litellm_input_audio_tokens_metric             (audio input tokens)
  - litellm_output_reasoning_tokens_metric        (reasoning tokens)
  - litellm_output_audio_tokens_metric            (audio output tokens)

These are additive — existing input/output/total counters are
unchanged, so no dashboards break. Each new counter is only
incremented when the underlying detail is populated and > 0, keeping
scrape output sparse for providers that don't report a given field.

Data is read from the canonical Usage dict that
`get_standard_logging_object_payload` already attaches at
`standard_logging_payload["metadata"]["usage_object"]`, so no new
plumbing through the logging pipeline is required.

Tests: 10 new unit tests covering registration, label-set parity,
all-types increment, zero/None/negative skip behaviour, and the
no-metadata/no-usage_object no-op paths.

Closes LIT-3220

Co-authored-by: shin-berri <shin-laptop@berri.ai>
Co-authored-by: yuneng-jiang <yuneng@berri.ai>
Co-authored-by: Krrish Dholakia <krrishdholakia@berri.ai>
Co-authored-by: Claude <noreply@anthropic.com>

* chore: remove proof folder image

---------

Co-authored-by: oss-agent-shin <ext-agent-shin@berri.ai>
Co-authored-by: shin-berri <shin-laptop@berri.ai>
Co-authored-by: yuneng-jiang <yuneng@berri.ai>
Co-authored-by: Krrish Dholakia <krrishdholakia@berri.ai>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Ishaan Jaffer <ishaanjaffer0324@gmail.com>
This commit is contained in:
ishaan-berri 2026-05-23 12:17:42 -07:00 committed by GitHub
parent 5e16f20962
commit 14c0a2b3e2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 418 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
litellm/integrations/prometheus.py
View File

@ -166,6 +166,53 @@ class PrometheusLogger(CustomLogger):
labelnames=self.get_labels_for_metric("litellm_output_tokens_metric"),
)
# Token-type detail metrics. These break out cached, cache-creation,
# audio and reasoning tokens that providers report inside
# prompt_tokens_details / completion_tokens_details on the usage
# object. They are sparse (only incremented when the provider
# reports a non-zero value) and are additive to the existing
# input/output token totals — no breaking change for existing
# dashboards built on the totals.
self.litellm_input_cached_tokens_metric = self._counter_factory(
"litellm_input_cached_tokens_metric",
"Provider-side cached input tokens (e.g. OpenAI prompt_tokens_details.cached_tokens, Anthropic cache_read_input_tokens)",
labelnames=self.get_labels_for_metric(
"litellm_input_cached_tokens_metric"
),
)
self.litellm_input_cache_creation_tokens_metric = self._counter_factory(
"litellm_input_cache_creation_tokens_metric",
"Provider-side input tokens written to prompt cache (e.g. Anthropic cache_creation_input_tokens)",
labelnames=self.get_labels_for_metric(
"litellm_input_cache_creation_tokens_metric"
),
)
self.litellm_input_audio_tokens_metric = self._counter_factory(
"litellm_input_audio_tokens_metric",
"Audio input tokens reported in prompt_tokens_details.audio_tokens",
labelnames=self.get_labels_for_metric(
"litellm_input_audio_tokens_metric"
),
)
self.litellm_output_reasoning_tokens_metric = self._counter_factory(
"litellm_output_reasoning_tokens_metric",
"Reasoning tokens reported in completion_tokens_details.reasoning_tokens",
labelnames=self.get_labels_for_metric(
"litellm_output_reasoning_tokens_metric"
),
)
self.litellm_output_audio_tokens_metric = self._counter_factory(
"litellm_output_audio_tokens_metric",
"Audio output tokens reported in completion_tokens_details.audio_tokens",
labelnames=self.get_labels_for_metric(
"litellm_output_audio_tokens_metric"
),
)
# Remaining Budget for Team
self.litellm_remaining_team_budget_metric = self._gauge_factory(
"litellm_remaining_team_budget_metric",
@ -1301,6 +1348,101 @@ class PrometheusLogger(CustomLogger):
amount=float(standard_logging_payload["completion_tokens"]),
)
# Token-type detail metrics — sparse, only emitted when the provider
# reports a non-zero value in usage.prompt_tokens_details /
# usage.completion_tokens_details.
self._increment_token_detail_metrics(
standard_logging_payload=standard_logging_payload,
enum_values=enum_values,
label_context=label_context,
)
def _increment_token_detail_metrics(
self,
standard_logging_payload: StandardLoggingPayload,
enum_values: UserAPIKeyLabelValues,
label_context: Optional[PrometheusLabelFactoryContext] = None,
) -> None:
"""
Increment per-token-type counters from the Usage object that providers
attach to the request. The Usage dict is plumbed onto
``standard_logging_payload["metadata"]["usage_object"]`` by
``get_standard_logging_object_payload``.
Each counter is only incremented when the underlying value is > 0, so
scrape output stays sparse for providers that don't report these
details (most non-OpenAI/Anthropic models).
"""
metadata = standard_logging_payload.get("metadata") or {}
usage_object = (
metadata.get("usage_object") if isinstance(metadata, dict) else None
)
if not isinstance(usage_object, dict):
return
prompt_details = usage_object.get("prompt_tokens_details") or {}
completion_details = usage_object.get("completion_tokens_details") or {}
detail_metrics: List[Tuple[Any, DEFINED_PROMETHEUS_METRICS, Any]] = [
(
self.litellm_input_cached_tokens_metric,
"litellm_input_cached_tokens_metric",
(
prompt_details.get("cached_tokens")
if isinstance(prompt_details, dict)
else None
),
),
(
self.litellm_input_cache_creation_tokens_metric,
"litellm_input_cache_creation_tokens_metric",
(
prompt_details.get("cache_creation_tokens")
if isinstance(prompt_details, dict)
else None
),
),
(
self.litellm_input_audio_tokens_metric,
"litellm_input_audio_tokens_metric",
(
prompt_details.get("audio_tokens")
if isinstance(prompt_details, dict)
else None
),
),
(
self.litellm_output_reasoning_tokens_metric,
"litellm_output_reasoning_tokens_metric",
(
completion_details.get("reasoning_tokens")
if isinstance(completion_details, dict)
else None
),
),
(
self.litellm_output_audio_tokens_metric,
"litellm_output_audio_tokens_metric",
(
completion_details.get("audio_tokens")
if isinstance(completion_details, dict)
else None
),
),
]
for counter, metric_name, value in detail_metrics:
if not isinstance(value, (int, float)) or value <= 0:
continue
PrometheusLogger._inc_labeled_counter(
self,
counter,
metric_name,
enum_values,
label_context=label_context,
amount=float(value),
)
def _increment_cache_metrics(
self,
standard_logging_payload: StandardLoggingPayload,

16
litellm/types/integrations/prometheus.py
View File

@ -201,6 +201,11 @@ DEFINED_PROMETHEUS_METRICS = Literal[
"litellm_total_tokens_metric",
"litellm_input_tokens_metric",
"litellm_output_tokens_metric",
"litellm_input_cached_tokens_metric",
"litellm_input_cache_creation_tokens_metric",
"litellm_input_audio_tokens_metric",
"litellm_output_reasoning_tokens_metric",
"litellm_output_audio_tokens_metric",
"litellm_deployment_successful_fallbacks",
"litellm_deployment_failed_fallbacks",
"litellm_remaining_team_budget_metric",
@ -451,6 +456,17 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.MODEL_ID.value,
]
# Token-type detail metrics — reuse the same label set as
# litellm_input_tokens_metric / litellm_output_tokens_metric so dashboards
# can join across them. Only emitted when the underlying usage detail is
# populated by the provider (e.g. Anthropic cache_read_input_tokens,
# OpenAI prompt_tokens_details.cached_tokens, reasoning_tokens, audio_tokens).
litellm_input_cached_tokens_metric = litellm_input_tokens_metric
litellm_input_cache_creation_tokens_metric = litellm_input_tokens_metric
litellm_input_audio_tokens_metric = litellm_input_tokens_metric
litellm_output_reasoning_tokens_metric = litellm_output_tokens_metric
litellm_output_audio_tokens_metric = litellm_output_tokens_metric
litellm_deployment_state = [
UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
UserAPIKeyLabelNames.MODEL_ID.value,

260
tests/test_litellm/integrations/test_prometheus_token_detail_metrics.py Normal file
View File

@ -0,0 +1,260 @@
"""
Unit tests for the per-token-type Prometheus detail metrics added for LIT-3220.
These metrics break out cached, cache-creation, audio and reasoning tokens
from the Usage object that providers report. They are sparse — only
incremented when the underlying detail is populated and > 0.
Run with:
uv run pytest tests/test_litellm/integrations/test_prometheus_token_detail_metrics.py -v
"""
from typing import get_args
from unittest.mock import MagicMock
import pytest
from litellm.integrations.prometheus import PrometheusLogger
from litellm.types.integrations.prometheus import (
DEFINED_PROMETHEUS_METRICS,
PrometheusMetricLabels,
UserAPIKeyLabelValues,
)
TOKEN_DETAIL_METRICS = [
"litellm_input_cached_tokens_metric",
"litellm_input_cache_creation_tokens_metric",
"litellm_input_audio_tokens_metric",
"litellm_output_reasoning_tokens_metric",
"litellm_output_audio_tokens_metric",
]
@pytest.fixture
def sample_enum_values():
return UserAPIKeyLabelValues(
end_user="test-end-user",
hashed_api_key="test-key-hash",
api_key_alias="test-key-alias",
team="test-team",
team_alias="test-team-alias",
user="test-user",
model="gpt-4o",
)
def _make_mock_logger():
"""Mock instance with the five detail counters + get_labels_for_metric."""
logger = MagicMock()
for name in TOKEN_DETAIL_METRICS:
setattr(logger, name, MagicMock())
logger.get_labels_for_metric = MagicMock(
return_value=[
"model",
"hashed_api_key",
"api_key_alias",
"team",
"team_alias",
"end_user",
"user",
]
)
return logger
class TestTokenDetailMetricsRegistration:
"""Metric registration / wiring — no runtime needed."""
def test_metrics_in_defined_prometheus_metrics(self):
defined = get_args(DEFINED_PROMETHEUS_METRICS)
for name in TOKEN_DETAIL_METRICS:
assert name in defined, f"{name} missing from DEFINED_PROMETHEUS_METRICS"
def test_metric_labels_defined(self):
for name in TOKEN_DETAIL_METRICS:
assert hasattr(
PrometheusMetricLabels, name
), f"{name} missing from PrometheusMetricLabels"
def test_input_detail_metrics_share_input_label_set(self):
# Detail metrics should reuse the parent input/output label set so
# dashboards can join token totals against per-type detail.
assert (
PrometheusMetricLabels.litellm_input_cached_tokens_metric
== PrometheusMetricLabels.litellm_input_tokens_metric
)
assert (
PrometheusMetricLabels.litellm_input_cache_creation_tokens_metric
== PrometheusMetricLabels.litellm_input_tokens_metric
)
assert (
PrometheusMetricLabels.litellm_input_audio_tokens_metric
== PrometheusMetricLabels.litellm_input_tokens_metric
)
def test_output_detail_metrics_share_output_label_set(self):
assert (
PrometheusMetricLabels.litellm_output_reasoning_tokens_metric
== PrometheusMetricLabels.litellm_output_tokens_metric
)
assert (
PrometheusMetricLabels.litellm_output_audio_tokens_metric
== PrometheusMetricLabels.litellm_output_tokens_metric
)
class TestIncrementTokenDetailMetrics:
"""Behaviour of PrometheusLogger._increment_token_detail_metrics."""
def test_increments_all_present_token_types(self, sample_enum_values):
logger = _make_mock_logger()
payload = {
"metadata": {
"usage_object": {
"prompt_tokens": 100,
"completion_tokens": 80,
"total_tokens": 180,
"prompt_tokens_details": {
"cached_tokens": 40,
"cache_creation_tokens": 25,
"audio_tokens": 15,
},
"completion_tokens_details": {
"reasoning_tokens": 60,
"audio_tokens": 10,
},
}
},
}
PrometheusLogger._increment_token_detail_metrics(
logger,
standard_logging_payload=payload,
enum_values=sample_enum_values,
)
logger.litellm_input_cached_tokens_metric.labels().inc.assert_called_once_with(
40.0
)
logger.litellm_input_cache_creation_tokens_metric.labels().inc.assert_called_once_with(
25.0
)
logger.litellm_input_audio_tokens_metric.labels().inc.assert_called_once_with(
15.0
)
logger.litellm_output_reasoning_tokens_metric.labels().inc.assert_called_once_with(
60.0
)
logger.litellm_output_audio_tokens_metric.labels().inc.assert_called_once_with(
10.0
)
def test_skips_metrics_when_value_is_zero(self, sample_enum_values):
logger = _make_mock_logger()
payload = {
"metadata": {
"usage_object": {
"prompt_tokens_details": {
"cached_tokens": 0,
"cache_creation_tokens": 0,
"audio_tokens": 0,
},
"completion_tokens_details": {
"reasoning_tokens": 0,
"audio_tokens": 0,
},
}
}
}
PrometheusLogger._increment_token_detail_metrics(
logger,
standard_logging_payload=payload,
enum_values=sample_enum_values,
)
for name in TOKEN_DETAIL_METRICS:
getattr(logger, name).labels.assert_not_called()
def test_skips_metrics_when_value_is_none(self, sample_enum_values):
logger = _make_mock_logger()
payload = {
"metadata": {
"usage_object": {
"prompt_tokens_details": {
"cached_tokens": None,
"audio_tokens": 12,
},
"completion_tokens_details": {},
}
}
}
PrometheusLogger._increment_token_detail_metrics(
logger,
standard_logging_payload=payload,
enum_values=sample_enum_values,
)
# Only audio_tokens was non-zero — only that counter should fire.
logger.litellm_input_cached_tokens_metric.labels.assert_not_called()
logger.litellm_input_cache_creation_tokens_metric.labels.assert_not_called()
logger.litellm_input_audio_tokens_metric.labels().inc.assert_called_once_with(
12.0
)
logger.litellm_output_reasoning_tokens_metric.labels.assert_not_called()
logger.litellm_output_audio_tokens_metric.labels.assert_not_called()
def test_no_usage_object_is_a_noop(self, sample_enum_values):
logger = _make_mock_logger()
payload = {"metadata": {}}
# Should not raise and should not call any counter.
PrometheusLogger._increment_token_detail_metrics(
logger,
standard_logging_payload=payload,
enum_values=sample_enum_values,
)
for name in TOKEN_DETAIL_METRICS:
getattr(logger, name).labels.assert_not_called()
def test_missing_metadata_is_a_noop(self, sample_enum_values):
logger = _make_mock_logger()
# Many error / cache-hit paths leave metadata as None.
PrometheusLogger._increment_token_detail_metrics(
logger,
standard_logging_payload={"metadata": None}, # type: ignore[typeddict-item]
enum_values=sample_enum_values,
)
for name in TOKEN_DETAIL_METRICS:
getattr(logger, name).labels.assert_not_called()
def test_negative_values_are_ignored(self, sample_enum_values):
# Defensive: a buggy upstream that returned a negative shouldn't
# poison the counter (counters can't go down without a reset).
logger = _make_mock_logger()
payload = {
"metadata": {
"usage_object": {
"prompt_tokens_details": {"cached_tokens": -5},
"completion_tokens_details": {"reasoning_tokens": -10},
}
}
}
PrometheusLogger._increment_token_detail_metrics(
logger,
standard_logging_payload=payload,
enum_values=sample_enum_values,
)
logger.litellm_input_cached_tokens_metric.labels.assert_not_called()
logger.litellm_output_reasoning_tokens_metric.labels.assert_not_called()
if __name__ == "__main__":
pytest.main([__file__, "-v"])