diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index 019410308c..0b754fa486 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -243,6 +243,115 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ ... "metadata_foo": "hello world" ... ``` + +## Configuring Metrics and Labels + +You can selectively enable specific metrics and control which labels are included to optimize performance and reduce cardinality. + +### Enable Specific Metrics and Labels + +Configure which metrics to emit by specifying them in `prometheus_metrics_config`. Each configuration group needs a `group` name (for organization) and a list of `metrics` to enable. You can optionally include a list of `include_labels` to filter the labels for the metrics. + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + +litellm_settings: + callbacks: ["prometheus"] + prometheus_metrics_config: + # High-cardinality metrics with minimal labels + - group: "proxy_metrics" + metrics: + - "litellm_proxy_total_requests_metric" + - "litellm_proxy_failed_requests_metric" + include_labels: + - "hashed_api_key" + - "requested_model" + - "model_group" +``` + +On starting up LiteLLM if your metrics were correctly configured, you should see the following on your container logs + + + + +### Filter Labels Per Metric + +Control which labels are included for each metric to reduce cardinality: + +```yaml +litellm_settings: + callbacks: ["prometheus"] + prometheus_metrics_config: + - group: "spend_and_tokens" + metrics: + - "litellm_spend_metric" + - "litellm_total_tokens" + include_labels: + - "model" + - "team" + - "hashed_api_key" + - group: "request_tracking" + metrics: + - "litellm_proxy_total_requests_metric" + include_labels: + - "status_code" + - "requested_model" +``` + +### Advanced Configuration + +You can create multiple configuration groups with different label sets: + +```yaml +litellm_settings: + callbacks: ["prometheus"] + prometheus_metrics_config: + # High-cardinality metrics with minimal labels + - group: "deployment_health" + metrics: + - "litellm_deployment_success_responses" + - "litellm_deployment_failure_responses" + include_labels: + - "api_provider" + - "requested_model" + + # Budget metrics with full label set + - group: "budget_tracking" + metrics: + - "litellm_spend_metric" + - "litellm_remaining_team_budget_metric" + include_labels: + - "team" + - "team_alias" + - "hashed_api_key" + - "api_key_alias" + - "model" + - "end_user" + + # Latency metrics with performance-focused labels + - group: "performance" + metrics: + - "litellm_request_total_latency_metric" + - "litellm_llm_api_latency_metric" + include_labels: + - "model" + - "api_provider" + - "requested_model" +``` + +**Configuration Structure:** +- `group`: A descriptive name for organizing related metrics +- `metrics`: List of metric names to include in this group +- `include_labels`: (Optional) List of labels to include for these metrics + +**Default Behavior**: If no `prometheus_metrics_config` is specified, all metrics are enabled with their default labels (backward compatible). + ## Monitor System Health To monitor the health of litellm adjacent services (redis / postgres), do: diff --git a/docs/my-website/img/prom_config.png b/docs/my-website/img/prom_config.png new file mode 100644 index 0000000000..b6ac6ecb16 Binary files /dev/null and b/docs/my-website/img/prom_config.png differ diff --git a/litellm/__init__.py b/litellm/__init__.py index 54b9798de9..27ed078fc5 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -309,6 +309,7 @@ disable_end_user_cost_tracking: Optional[bool] = None disable_end_user_cost_tracking_prometheus_only: Optional[bool] = None enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None custom_prometheus_metadata_labels: List[str] = [] +prometheus_metrics_config: Optional[List] = None disable_add_prefix_to_prompt: bool = ( False # used by anthropic, to disable adding prefix to prompt ) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 96ba23fb9c..4de47425d5 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -8,6 +8,7 @@ from typing import ( Any, Awaitable, Callable, + Dict, List, Literal, Optional, @@ -50,42 +51,48 @@ class PrometheusLogger(CustomLogger): ) return - self.litellm_proxy_failed_requests_metric = Counter( + # Parse prometheus metrics configuration for label filtering + self.label_filters = self._parse_prometheus_config() + + # Create metric factory functions + self._counter_factory = self._create_metric_factory(Counter) + self._gauge_factory = self._create_metric_factory(Gauge) + self._histogram_factory = self._create_metric_factory(Histogram) + + self.litellm_proxy_failed_requests_metric = self._counter_factory( name="litellm_proxy_failed_requests_metric", documentation="Total number of failed responses from proxy - the client did not get a success response from litellm proxy", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_proxy_failed_requests_metric" + labelnames=self.get_labels_for_metric( + "litellm_proxy_failed_requests_metric" ), ) - self.litellm_proxy_total_requests_metric = Counter( + self.litellm_proxy_total_requests_metric = self._counter_factory( name="litellm_proxy_total_requests_metric", documentation="Total number of requests made to the proxy server - track number of client side requests", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_proxy_total_requests_metric" + labelnames=self.get_labels_for_metric( + "litellm_proxy_total_requests_metric" ), ) # request latency metrics - self.litellm_request_total_latency_metric = Histogram( + self.litellm_request_total_latency_metric = self._histogram_factory( "litellm_request_total_latency_metric", "Total latency (seconds) for a request to LiteLLM", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_request_total_latency_metric" + labelnames=self.get_labels_for_metric( + "litellm_request_total_latency_metric" ), buckets=LATENCY_BUCKETS, ) - self.litellm_llm_api_latency_metric = Histogram( + self.litellm_llm_api_latency_metric = self._histogram_factory( "litellm_llm_api_latency_metric", "Total latency (seconds) for a models LLM API call", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_llm_api_latency_metric" - ), + labelnames=self.get_labels_for_metric("litellm_llm_api_latency_metric"), buckets=LATENCY_BUCKETS, ) - self.litellm_llm_api_time_to_first_token_metric = Histogram( + self.litellm_llm_api_time_to_first_token_metric = self._histogram_factory( "litellm_llm_api_time_to_first_token_metric", "Time to first token for a models LLM API call", labelnames=[ @@ -99,7 +106,7 @@ class PrometheusLogger(CustomLogger): ) # Counter for spend - self.litellm_spend_metric = Counter( + self.litellm_spend_metric = self._counter_factory( "litellm_spend_metric", "Total spend on LLM requests", labelnames=[ @@ -114,80 +121,72 @@ class PrometheusLogger(CustomLogger): ) # Counter for total_output_tokens - self.litellm_tokens_metric = Counter( + self.litellm_tokens_metric = self._counter_factory( "litellm_total_tokens", "Total number of input + output tokens from LLM requests", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_total_tokens_metric" - ), + labelnames=self.get_labels_for_metric("litellm_total_tokens_metric"), ) - self.litellm_input_tokens_metric = Counter( + self.litellm_input_tokens_metric = self._counter_factory( "litellm_input_tokens", "Total number of input tokens from LLM requests", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_input_tokens_metric" - ), + labelnames=self.get_labels_for_metric("litellm_input_tokens_metric"), ) - self.litellm_output_tokens_metric = Counter( + self.litellm_output_tokens_metric = self._counter_factory( "litellm_output_tokens", "Total number of output tokens from LLM requests", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_output_tokens_metric" - ), + labelnames=self.get_labels_for_metric("litellm_output_tokens_metric"), ) # Remaining Budget for Team - self.litellm_remaining_team_budget_metric = Gauge( + self.litellm_remaining_team_budget_metric = self._gauge_factory( "litellm_remaining_team_budget_metric", "Remaining budget for team", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_remaining_team_budget_metric" + labelnames=self.get_labels_for_metric( + "litellm_remaining_team_budget_metric" ), ) # Max Budget for Team - self.litellm_team_max_budget_metric = Gauge( + self.litellm_team_max_budget_metric = self._gauge_factory( "litellm_team_max_budget_metric", "Maximum budget set for team", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_team_max_budget_metric" - ), + labelnames=self.get_labels_for_metric("litellm_team_max_budget_metric"), ) # Team Budget Reset At - self.litellm_team_budget_remaining_hours_metric = Gauge( + self.litellm_team_budget_remaining_hours_metric = self._gauge_factory( "litellm_team_budget_remaining_hours_metric", "Remaining days for team budget to be reset", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_team_budget_remaining_hours_metric" + labelnames=self.get_labels_for_metric( + "litellm_team_budget_remaining_hours_metric" ), ) # Remaining Budget for API Key - self.litellm_remaining_api_key_budget_metric = Gauge( + self.litellm_remaining_api_key_budget_metric = self._gauge_factory( "litellm_remaining_api_key_budget_metric", "Remaining budget for api key", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_remaining_api_key_budget_metric" + labelnames=self.get_labels_for_metric( + "litellm_remaining_api_key_budget_metric" ), ) # Max Budget for API Key - self.litellm_api_key_max_budget_metric = Gauge( + self.litellm_api_key_max_budget_metric = self._gauge_factory( "litellm_api_key_max_budget_metric", "Maximum budget set for api key", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_api_key_max_budget_metric" + labelnames=self.get_labels_for_metric( + "litellm_api_key_max_budget_metric" ), ) - self.litellm_api_key_budget_remaining_hours_metric = Gauge( + self.litellm_api_key_budget_remaining_hours_metric = self._gauge_factory( "litellm_api_key_budget_remaining_hours_metric", "Remaining hours for api key budget to be reset", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_api_key_budget_remaining_hours_metric" + labelnames=self.get_labels_for_metric( + "litellm_api_key_budget_remaining_hours_metric" ), ) @@ -195,14 +194,14 @@ class PrometheusLogger(CustomLogger): # LiteLLM Virtual API KEY metrics ######################################## # Remaining MODEL RPM limit for API Key - self.litellm_remaining_api_key_requests_for_model = Gauge( + self.litellm_remaining_api_key_requests_for_model = self._gauge_factory( "litellm_remaining_api_key_requests_for_model", "Remaining Requests API Key can make for model (model based rpm limit on key)", labelnames=["hashed_api_key", "api_key_alias", "model"], ) # Remaining MODEL TPM limit for API Key - self.litellm_remaining_api_key_tokens_for_model = Gauge( + self.litellm_remaining_api_key_tokens_for_model = self._gauge_factory( "litellm_remaining_api_key_tokens_for_model", "Remaining Tokens API Key can make for model (model based tpm limit on key)", labelnames=["hashed_api_key", "api_key_alias", "model"], @@ -213,7 +212,7 @@ class PrometheusLogger(CustomLogger): ######################################## # Remaining Rate Limit for model - self.litellm_remaining_requests_metric = Gauge( + self.litellm_remaining_requests_metric = self._gauge_factory( "litellm_remaining_requests", "LLM Deployment Analytics - remaining requests for model, returned from LLM API Provider", labelnames=[ @@ -226,7 +225,7 @@ class PrometheusLogger(CustomLogger): ], ) - self.litellm_remaining_tokens_metric = Gauge( + self.litellm_remaining_tokens_metric = self._gauge_factory( "litellm_remaining_tokens", "remaining tokens for model, returned from LLM API Provider", labelnames=[ @@ -239,7 +238,7 @@ class PrometheusLogger(CustomLogger): ], ) - self.litellm_overhead_latency_metric = Histogram( + self.litellm_overhead_latency_metric = self._histogram_factory( "litellm_overhead_latency_metric", "Latency overhead (milliseconds) added by LiteLLM processing", labelnames=[ @@ -253,7 +252,7 @@ class PrometheusLogger(CustomLogger): buckets=LATENCY_BUCKETS, ) # llm api provider budget metrics - self.litellm_provider_remaining_budget_metric = Gauge( + self.litellm_provider_remaining_budget_metric = self._gauge_factory( "litellm_provider_remaining_budget_metric", "Remaining budget for provider - used when you set provider budget limits", labelnames=["api_provider"], @@ -266,40 +265,35 @@ class PrometheusLogger(CustomLogger): UserAPIKeyLabelNames.API_BASE.value, UserAPIKeyLabelNames.API_PROVIDER.value, ] - team_and_key_labels = [ - "hashed_api_key", - "api_key_alias", - "team", - "team_alias", - ] # Metric for deployment state - self.litellm_deployment_state = Gauge( + self.litellm_deployment_state = self._gauge_factory( "litellm_deployment_state", "LLM Deployment Analytics - The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outage", labelnames=_logged_llm_labels, ) - self.litellm_deployment_cooled_down = Counter( + self.litellm_deployment_cooled_down = self._counter_factory( "litellm_deployment_cooled_down", "LLM Deployment Analytics - Number of times a deployment has been cooled down by LiteLLM load balancing logic. exception_status is the status of the exception that caused the deployment to be cooled down", labelnames=_logged_llm_labels + [EXCEPTION_STATUS], ) - self.litellm_deployment_success_responses = Counter( + self.litellm_deployment_success_responses = self._counter_factory( name="litellm_deployment_success_responses", documentation="LLM Deployment Analytics - Total number of successful LLM API calls via litellm", - labelnames=[REQUESTED_MODEL] + _logged_llm_labels + team_and_key_labels, + labelnames=self.get_labels_for_metric( + "litellm_deployment_success_responses" + ), ) - self.litellm_deployment_failure_responses = Counter( + self.litellm_deployment_failure_responses = self._counter_factory( name="litellm_deployment_failure_responses", documentation="LLM Deployment Analytics - Total number of failed LLM API calls for a specific LLM deploymeny. exception_status is the status of the exception from the llm api", - labelnames=[REQUESTED_MODEL] - + _logged_llm_labels - + EXCEPTION_LABELS - + team_and_key_labels, + labelnames=self.get_labels_for_metric( + "litellm_deployment_failure_responses" + ), ) - self.litellm_deployment_failure_by_tag_responses = Counter( + self.litellm_deployment_failure_by_tag_responses = self._counter_factory( "litellm_deployment_failure_by_tag_responses", "Total number of failed LLM API calls for a specific LLM deploymeny by custom metadata tags", labelnames=[ @@ -309,44 +303,36 @@ class PrometheusLogger(CustomLogger): + _logged_llm_labels + EXCEPTION_LABELS, ) - self.litellm_deployment_total_requests = Counter( + self.litellm_deployment_total_requests = self._counter_factory( name="litellm_deployment_total_requests", documentation="LLM Deployment Analytics - Total number of LLM API calls via litellm - success + failure", - labelnames=[REQUESTED_MODEL] + _logged_llm_labels + team_and_key_labels, + labelnames=self.get_labels_for_metric( + "litellm_deployment_total_requests" + ), ) # Deployment Latency tracking - team_and_key_labels = [ - "hashed_api_key", - "api_key_alias", - "team", - "team_alias", - ] - self.litellm_deployment_latency_per_output_token = Histogram( + self.litellm_deployment_latency_per_output_token = self._histogram_factory( name="litellm_deployment_latency_per_output_token", documentation="LLM Deployment Analytics - Latency per output token", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_deployment_latency_per_output_token" + labelnames=self.get_labels_for_metric( + "litellm_deployment_latency_per_output_token" ), ) - self.litellm_deployment_successful_fallbacks = Counter( + self.litellm_deployment_successful_fallbacks = self._counter_factory( "litellm_deployment_successful_fallbacks", "LLM Deployment Analytics - Number of successful fallback requests from primary model -> fallback model", - PrometheusMetricLabels.get_labels( - "litellm_deployment_successful_fallbacks" - ), + self.get_labels_for_metric("litellm_deployment_successful_fallbacks"), ) - self.litellm_deployment_failed_fallbacks = Counter( + self.litellm_deployment_failed_fallbacks = self._counter_factory( "litellm_deployment_failed_fallbacks", "LLM Deployment Analytics - Number of failed fallback requests from primary model -> fallback model", - PrometheusMetricLabels.get_labels( - "litellm_deployment_failed_fallbacks" - ), + self.get_labels_for_metric("litellm_deployment_failed_fallbacks"), ) - self.litellm_llm_api_failed_requests_metric = Counter( + self.litellm_llm_api_failed_requests_metric = self._counter_factory( name="litellm_llm_api_failed_requests_metric", documentation="deprecated - use litellm_proxy_failed_requests_metric", labelnames=[ @@ -360,17 +346,171 @@ class PrometheusLogger(CustomLogger): ], ) - self.litellm_requests_metric = Counter( + self.litellm_requests_metric = self._counter_factory( name="litellm_requests_metric", documentation="deprecated - use litellm_proxy_total_requests_metric. Total number of LLM calls to litellm - track total per API Key, team, user", - labelnames=PrometheusMetricLabels.get_labels( - label_name="litellm_requests_metric" - ), + labelnames=self.get_labels_for_metric("litellm_requests_metric"), ) except Exception as e: print_verbose(f"Got exception on init prometheus client {str(e)}") raise e + def _parse_prometheus_config(self) -> Dict[str, List[str]]: + """Parse prometheus metrics configuration for label filtering and enabled metrics""" + import litellm + from litellm.types.integrations.prometheus import PrometheusMetricsConfig + + config = litellm.prometheus_metrics_config + + # If no config is provided, return empty dict (no filtering) + if not config: + return {} + + verbose_logger.debug(f"prometheus config: {config}") + + label_filters = {} + self.enabled_metrics = set() + + # Parse each configuration group + for group_config in config: + # Validate configuration using Pydantic + if isinstance(group_config, dict): + parsed_config = PrometheusMetricsConfig(**group_config) + else: + parsed_config = group_config + + # Add enabled metrics to the set + self.enabled_metrics.update(parsed_config.metrics) + + # Set label filters for each metric in this group + for metric_name in parsed_config.metrics: + if parsed_config.include_labels: + label_filters[metric_name] = parsed_config.include_labels + + # Pretty print the processed configuration + self._pretty_print_prometheus_config(label_filters) + + return label_filters + + def _pretty_print_prometheus_config( + self, label_filters: Dict[str, List[str]] + ) -> None: + """Pretty print the processed prometheus configuration using rich""" + try: + from rich.console import Console + from rich.panel import Panel + from rich.table import Table + from rich.text import Text + + console = Console() + + # Create main panel title + title = Text("Prometheus Configuration Processed", style="bold blue") + + # Create enabled metrics table + metrics_table = Table( + title="📊 Enabled Metrics", + show_header=True, + header_style="bold magenta", + title_justify="left", + ) + metrics_table.add_column("Metric Name", style="cyan", no_wrap=True) + + if hasattr(self, "enabled_metrics") and self.enabled_metrics: + for metric in sorted(self.enabled_metrics): + metrics_table.add_row(metric) + else: + metrics_table.add_row( + "[yellow]All metrics enabled (no filter applied)[/yellow]" + ) + + # Create label filters table + labels_table = Table( + title="🏷️ Label Filters", + show_header=True, + header_style="bold green", + title_justify="left", + ) + labels_table.add_column("Metric Name", style="cyan", no_wrap=True) + labels_table.add_column("Allowed Labels", style="yellow") + + if label_filters: + for metric_name, labels in sorted(label_filters.items()): + labels_str = ( + ", ".join(labels) + if labels + else "[dim]No labels specified[/dim]" + ) + labels_table.add_row(metric_name, labels_str) + else: + labels_table.add_row( + "[yellow]No label filtering applied[/yellow]", + "[dim]All default labels will be used[/dim]", + ) + + # Print everything in a nice panel + console.print("\n") + console.print(Panel(title, border_style="blue")) + console.print(metrics_table) + console.print(labels_table) + console.print("\n") + + except ImportError: + # Fallback to simple logging if rich is not available + verbose_logger.info( + f"Enabled metrics: {sorted(self.enabled_metrics) if hasattr(self, 'enabled_metrics') else 'All metrics'}" + ) + verbose_logger.info(f"Label filters: {label_filters}") + + def _is_metric_enabled(self, metric_name: str) -> bool: + """Check if a metric is enabled based on configuration""" + # If no specific configuration is provided, enable all metrics (default behavior) + if not hasattr(self, "enabled_metrics"): + return True + + # If enabled_metrics is empty, enable all metrics + if not self.enabled_metrics: + return True + + return metric_name in self.enabled_metrics + + def _create_metric_factory(self, metric_class): + """Create a factory function that returns either a real metric or a no-op metric""" + + def factory(*args, **kwargs): + # Extract metric name from the first argument or 'name' keyword argument + metric_name = args[0] if args else kwargs.get("name", "") + + if self._is_metric_enabled(metric_name): + return metric_class(*args, **kwargs) + else: + return NoOpMetric() + + return factory + + def get_labels_for_metric( + self, metric_name: DEFINED_PROMETHEUS_METRICS + ) -> List[str]: + """ + Get the labels for a metric, filtered if configured + """ + # Get default labels for this metric from PrometheusMetricLabels + default_labels = PrometheusMetricLabels.get_labels(metric_name) + + # If no label filtering is configured for this metric, use default labels + if metric_name not in self.label_filters: + return default_labels + + # Get configured labels for this metric + configured_labels = self.label_filters[metric_name] + + # Return intersection of configured and default labels to ensure we only use valid labels + filtered_labels = [ + label for label in default_labels if label in configured_labels + ] + + return filtered_labels + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): # Define prometheus client from litellm.types.utils import StandardLoggingPayload @@ -524,8 +664,8 @@ class PrometheusLogger(CustomLogger): standard_logging_payload["stream"] is True ): # log successful streaming requests from logging event hook. _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_proxy_total_requests_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_proxy_total_requests_metric" ), enum_values=enum_values, ) @@ -552,15 +692,15 @@ class PrometheusLogger(CustomLogger): _tags = standard_logging_payload["request_tags"] _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_proxy_total_requests_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_proxy_total_requests_metric" ), enum_values=enum_values, ) _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_total_tokens_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_total_tokens_metric" ), enum_values=enum_values, ) @@ -569,8 +709,8 @@ class PrometheusLogger(CustomLogger): ) _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_input_tokens_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_input_tokens_metric" ), enum_values=enum_values, ) @@ -579,8 +719,8 @@ class PrometheusLogger(CustomLogger): ) _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_output_tokens_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_output_tokens_metric" ), enum_values=enum_values, ) @@ -640,8 +780,8 @@ class PrometheusLogger(CustomLogger): enum_values: UserAPIKeyLabelValues, ): _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_requests_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_requests_metric" ), enum_values=enum_values, ) @@ -649,8 +789,8 @@ class PrometheusLogger(CustomLogger): self.litellm_requests_metric.labels(**_labels).inc() _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_proxy_total_requests_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_proxy_total_requests_metric" ), enum_values=enum_values, ) @@ -742,8 +882,8 @@ class PrometheusLogger(CustomLogger): ) if api_call_total_time_seconds is not None: _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_llm_api_latency_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_llm_api_latency_metric" ), enum_values=enum_values, ) @@ -758,8 +898,8 @@ class PrometheusLogger(CustomLogger): ) if total_time_seconds is not None: _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_request_total_latency_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_request_total_latency_metric" ), enum_values=enum_values, ) @@ -849,16 +989,16 @@ class PrometheusLogger(CustomLogger): route=user_api_key_dict.request_route, ) _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_proxy_failed_requests_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_proxy_failed_requests_metric" ), enum_values=enum_values, ) self.litellm_proxy_failed_requests_metric.labels(**_labels).inc() _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_proxy_total_requests_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_proxy_total_requests_metric" ), enum_values=enum_values, ) @@ -890,8 +1030,8 @@ class PrometheusLogger(CustomLogger): route=user_api_key_dict.request_route, ) _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_proxy_total_requests_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_proxy_total_requests_metric" ), enum_values=enum_values, ) @@ -925,7 +1065,7 @@ class PrometheusLogger(CustomLogger): model_group = standard_logging_payload.get("model_group", None) api_base = standard_logging_payload.get("api_base", None) model_id = standard_logging_payload.get("model_id", None) - exception: Exception = request_kwargs.get("exception", None) + exception = request_kwargs.get("exception", None) llm_provider = _litellm_params.get("custom_llm_provider", None) @@ -934,51 +1074,52 @@ class PrometheusLogger(CustomLogger): ["litellm_model_name", "model_id", "api_base", "api_provider"] """ self.set_deployment_partial_outage( - litellm_model_name=litellm_model_name, + litellm_model_name=litellm_model_name or "", model_id=model_id, api_base=api_base, - api_provider=llm_provider, + api_provider=llm_provider or "", ) - self.litellm_deployment_failure_responses.labels( - litellm_model_name=litellm_model_name, - model_id=model_id, - api_base=api_base, - api_provider=llm_provider, - exception_status=str(getattr(exception, "status_code", None)), - exception_class=self._get_exception_class_name(exception), - requested_model=model_group, - hashed_api_key=standard_logging_payload["metadata"][ - "user_api_key_hash" - ], - api_key_alias=standard_logging_payload["metadata"][ - "user_api_key_alias" - ], - team=standard_logging_payload["metadata"]["user_api_key_team_id"], - team_alias=standard_logging_payload["metadata"][ - "user_api_key_team_alias" - ], - ).inc() + if exception is not None: + self.litellm_deployment_failure_responses.labels( + litellm_model_name=litellm_model_name, + model_id=model_id, + api_base=api_base, + api_provider=llm_provider, + exception_status=str(getattr(exception, "status_code", None)), + exception_class=self._get_exception_class_name(exception), + requested_model=model_group, + hashed_api_key=standard_logging_payload["metadata"][ + "user_api_key_hash" + ], + api_key_alias=standard_logging_payload["metadata"][ + "user_api_key_alias" + ], + team=standard_logging_payload["metadata"]["user_api_key_team_id"], + team_alias=standard_logging_payload["metadata"][ + "user_api_key_team_alias" + ], + ).inc() - # tag based tracking - if standard_logging_payload is not None and isinstance( - standard_logging_payload, dict - ): - _tags = standard_logging_payload["request_tags"] - for tag in _tags: - self.litellm_deployment_failure_by_tag_responses.labels( - **{ - UserAPIKeyLabelNames.REQUESTED_MODEL.value: model_group, - UserAPIKeyLabelNames.TAG.value: tag, - UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value: litellm_model_name, - UserAPIKeyLabelNames.MODEL_ID.value: model_id, - UserAPIKeyLabelNames.API_BASE.value: api_base, - UserAPIKeyLabelNames.API_PROVIDER.value: llm_provider, - UserAPIKeyLabelNames.EXCEPTION_CLASS.value: exception.__class__.__name__, - UserAPIKeyLabelNames.EXCEPTION_STATUS.value: str( - getattr(exception, "status_code", None) - ), - } - ).inc() + # tag based tracking + if standard_logging_payload is not None and isinstance( + standard_logging_payload, dict + ): + _tags = standard_logging_payload["request_tags"] + for tag in _tags: + self.litellm_deployment_failure_by_tag_responses.labels( + **{ + UserAPIKeyLabelNames.REQUESTED_MODEL.value: model_group, + UserAPIKeyLabelNames.TAG.value: tag, + UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value: litellm_model_name, + UserAPIKeyLabelNames.MODEL_ID.value: model_id, + UserAPIKeyLabelNames.API_BASE.value: api_base, + UserAPIKeyLabelNames.API_PROVIDER.value: llm_provider, + UserAPIKeyLabelNames.EXCEPTION_CLASS.value: exception.__class__.__name__, + UserAPIKeyLabelNames.EXCEPTION_STATUS.value: str( + getattr(exception, "status_code", None) + ), + } + ).inc() self.litellm_deployment_total_requests.labels( litellm_model_name=litellm_model_name, @@ -1016,9 +1157,9 @@ class PrometheusLogger(CustomLogger): ): try: verbose_logger.debug("setting remaining tokens requests metric") - standard_logging_payload: Optional[ - StandardLoggingPayload - ] = request_kwargs.get("standard_logging_object") + standard_logging_payload: Optional[StandardLoggingPayload] = ( + request_kwargs.get("standard_logging_object") + ) if standard_logging_payload is None: return @@ -1091,10 +1232,10 @@ class PrometheusLogger(CustomLogger): ["litellm_model_name", "requested_model", model_id", "api_base", "api_provider"] """ self.set_deployment_healthy( - litellm_model_name=litellm_model_name, - model_id=model_id, - api_base=api_base, - api_provider=llm_provider, + litellm_model_name=litellm_model_name or "", + model_id=model_id or "", + api_base=api_base or "", + api_provider=llm_provider or "", ) self.litellm_deployment_success_responses.labels( @@ -1157,8 +1298,8 @@ class PrometheusLogger(CustomLogger): if output_tokens is not None and output_tokens > 0: latency_per_token = _latency_seconds / output_tokens _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_deployment_latency_per_output_token" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_deployment_latency_per_output_token" ), enum_values=enum_values, ) @@ -1229,8 +1370,8 @@ class PrometheusLogger(CustomLogger): tags=_tags, ) _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_deployment_successful_fallbacks" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_deployment_successful_fallbacks" ), enum_values=enum_values, ) @@ -1274,8 +1415,8 @@ class PrometheusLogger(CustomLogger): ) _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_deployment_failed_fallbacks" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_deployment_failed_fallbacks" ), enum_values=enum_values, ) @@ -1622,8 +1763,8 @@ class PrometheusLogger(CustomLogger): ) _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_remaining_team_budget_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_remaining_team_budget_metric" ), enum_values=enum_values, ) @@ -1636,8 +1777,8 @@ class PrometheusLogger(CustomLogger): if team.max_budget is not None: _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_team_max_budget_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_team_max_budget_metric" ), enum_values=enum_values, ) @@ -1645,8 +1786,8 @@ class PrometheusLogger(CustomLogger): if team.budget_reset_at is not None: _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_team_budget_remaining_hours_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_team_budget_remaining_hours_metric" ), enum_values=enum_values, ) @@ -1669,8 +1810,8 @@ class PrometheusLogger(CustomLogger): api_key_alias=user_api_key_dict.key_alias or "", ) _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_remaining_api_key_budget_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_remaining_api_key_budget_metric" ), enum_values=enum_values, ) @@ -1683,8 +1824,8 @@ class PrometheusLogger(CustomLogger): if user_api_key_dict.max_budget is not None: _labels = prometheus_label_factory( - supported_enum_labels=PrometheusMetricLabels.get_labels( - label_name="litellm_api_key_max_budget_metric" + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_api_key_max_budget_metric" ), enum_values=enum_values, ) @@ -1788,10 +1929,10 @@ class PrometheusLogger(CustomLogger): from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.prometheus import PrometheusLogger - prometheus_loggers: List[ - CustomLogger - ] = litellm.logging_callback_manager.get_custom_loggers_for_type( - callback_type=PrometheusLogger + prometheus_loggers: List[CustomLogger] = ( + litellm.logging_callback_manager.get_custom_loggers_for_type( + callback_type=PrometheusLogger + ) ) # we need to get the initialized prometheus logger instance(s) and call logger.initialize_remaining_budget_metrics() on them verbose_logger.debug("found %s prometheus loggers", len(prometheus_loggers)) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 1dbae86277..29d07fa12b 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -7,3 +7,16 @@ model_list: litellm_params: model: "anthropic/*" api_key: os.environ/ANTHROPIC_API_KEY + +litellm_settings: + callbacks: ["prometheus"] + prometheus_metrics_config: + # High-cardinality metrics with minimal labels + - group: "proxy_metrics" + metrics: + - "litellm_proxy_total_requests_metric" + - "litellm_proxy_failed_requests_metric" + include_labels: + - "hashed_api_key" + - "requested_model" + - "model_group" \ No newline at end of file diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 49009f6e95..13714871b8 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -91,6 +91,9 @@ DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_remaining_api_key_budget_metric", "litellm_api_key_max_budget_metric", "litellm_api_key_budget_remaining_hours_metric", + "litellm_deployment_failure_responses", + "litellm_deployment_total_requests", + "litellm_deployment_success_responses", ] @@ -237,6 +240,35 @@ class PrometheusMetricLabels: litellm_remaining_api_key_budget_metric ) + # Add deployment metrics + litellm_deployment_failure_responses = [ + UserAPIKeyLabelNames.REQUESTED_MODEL.value, + UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, + UserAPIKeyLabelNames.MODEL_ID.value, + UserAPIKeyLabelNames.API_BASE.value, + UserAPIKeyLabelNames.API_PROVIDER.value, + UserAPIKeyLabelNames.EXCEPTION_STATUS.value, + UserAPIKeyLabelNames.EXCEPTION_CLASS.value, + UserAPIKeyLabelNames.API_KEY_HASH.value, + UserAPIKeyLabelNames.API_KEY_ALIAS.value, + UserAPIKeyLabelNames.TEAM.value, + UserAPIKeyLabelNames.TEAM_ALIAS.value, + ] + + litellm_deployment_total_requests = [ + UserAPIKeyLabelNames.REQUESTED_MODEL.value, + UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, + UserAPIKeyLabelNames.MODEL_ID.value, + UserAPIKeyLabelNames.API_BASE.value, + UserAPIKeyLabelNames.API_PROVIDER.value, + UserAPIKeyLabelNames.API_KEY_HASH.value, + UserAPIKeyLabelNames.API_KEY_ALIAS.value, + UserAPIKeyLabelNames.TEAM.value, + UserAPIKeyLabelNames.TEAM_ALIAS.value, + ] + + litellm_deployment_success_responses = litellm_deployment_total_requests + @staticmethod def get_labels(label_name: DEFINED_PROMETHEUS_METRICS) -> List[str]: default_labels = getattr(PrometheusMetricLabels, label_name) @@ -310,3 +342,44 @@ class UserAPIKeyLabelValues(BaseModel): route: Annotated[ Optional[str], Field(..., alias=UserAPIKeyLabelNames.ROUTE.value) ] = None + + +class PrometheusMetricsConfig(BaseModel): + """Configuration for filtering Prometheus metrics""" + + group: str = Field(..., description="Group name for this set of metrics") + metrics: List[str] = Field( + ..., description="List of metric names to include in this group" + ) + include_labels: Optional[List[str]] = Field( + None, + description="List of labels to include for these metrics. If None, includes all default labels.", + ) + + +class PrometheusSettings(BaseModel): + """Settings for Prometheus metrics configuration""" + + prometheus_metrics_config: Optional[List[PrometheusMetricsConfig]] = Field( + None, + description="Configuration for filtering Prometheus metrics by groups and labels", + ) + + +class NoOpMetric: + """A no-op metric that has the same interface as prometheus metrics but does nothing""" + + def __init__(self, *args, **kwargs): + pass + + def labels(self, *args, **kwargs): + return self + + def inc(self, *args, **kwargs): + pass + + def set(self, *args, **kwargs): + pass + + def observe(self, *args, **kwargs): + pass diff --git a/tests/test_litellm/integrations/test_prometheus.py b/tests/test_litellm/integrations/test_prometheus.py index fe57e4fc7b..2548eec2fc 100644 --- a/tests/test_litellm/integrations/test_prometheus.py +++ b/tests/test_litellm/integrations/test_prometheus.py @@ -17,6 +17,8 @@ from unittest.mock import patch import pytest_asyncio from apscheduler.schedulers.asyncio import AsyncIOScheduler + +# Add prometheus_client import for registry cleanup from prometheus_client import REGISTRY import litellm @@ -24,11 +26,34 @@ from litellm.constants import PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES from litellm.integrations.prometheus import PrometheusLogger, prometheus_label_factory from litellm.types.integrations.prometheus import ( PrometheusMetricLabels, + PrometheusMetricsConfig, UserAPIKeyLabelValues, ) +@pytest.fixture +def prometheus_logger() -> PrometheusLogger: + """ + Fixture that creates a clean PrometheusLogger instance by clearing the registry first. + This prevents "Duplicated timeseries in CollectorRegistry" errors. + """ + collectors = list(REGISTRY._collector_to_names.keys()) + for collector in collectors: + REGISTRY.unregister(collector) + return PrometheusLogger() + + +def clear_prometheus_registry(): + """Helper function to clear the Prometheus registry""" + collectors = list(REGISTRY._collector_to_names.keys()) + for collector in collectors: + REGISTRY.unregister(collector) + + def test_initialize_budget_metrics_cron_job(): + # Clear registry before test + clear_prometheus_registry() + # Create a scheduler scheduler = AsyncIOScheduler() @@ -189,6 +214,167 @@ def test_future_metrics_with_end_user_are_filtered(): litellm.enable_end_user_cost_tracking_prometheus_only = original_setting +def test_prometheus_config_parsing(): + """Test that prometheus metrics configuration is parsed correctly""" + # Clear registry before test + clear_prometheus_registry() + + # Set up test configuration + test_config = [ + { + "group": "service_metrics", + "metrics": [ + "litellm_deployment_failure_responses", + "litellm_deployment_total_requests", + "litellm_proxy_failed_requests_metric", + "litellm_proxy_total_requests_metric", + ], + "include_labels": [ + "litellm_model_name", + "requested_model", + "api_base", + "api_provider", + "exception_status", + "exception_class", + ], + } + ] + + # Set configuration + litellm.prometheus_metrics_config = test_config + + # Create PrometheusLogger instance + logger = PrometheusLogger() + + # Parse configuration + label_filters = logger._parse_prometheus_config() + + # Verify label filters exist for each metric + expected_labels = [ + "litellm_model_name", + "requested_model", + "api_base", + "api_provider", + "exception_status", + "exception_class", + ] + + expected_metrics = [ + "litellm_deployment_failure_responses", + "litellm_deployment_total_requests", + "litellm_proxy_failed_requests_metric", + "litellm_proxy_total_requests_metric", + ] + + for metric in expected_metrics: + assert metric in label_filters + assert label_filters[metric] == expected_labels + + +def test_get_metric_labels(): + """Test that metric label filtering works correctly""" + # Clear registry before test + clear_prometheus_registry() + + # Set up test configuration + test_config = [ + { + "group": "service_metrics", + "metrics": ["litellm_deployment_failure_responses"], + "include_labels": ["litellm_model_name", "api_provider"], + } + ] + + litellm.prometheus_metrics_config = test_config + + logger = PrometheusLogger() + + # Get filtered labels + labels = logger.get_labels_for_metric("litellm_deployment_failure_responses") + + # Verify only configured labels are returned + assert "litellm_model_name" in labels + assert "api_provider" in labels + # These should be filtered out even if they're in the default labels + assert ( + len([l for l in labels if l not in ["litellm_model_name", "api_provider"]]) == 0 + ) + + +def test_no_prometheus_config(): + """Test behavior when no prometheus config is set""" + # Clear registry before test + clear_prometheus_registry() + + # Clear any existing config + litellm.prometheus_metrics_config = None + + logger = PrometheusLogger() + + # Should return default labels when no config is set + labels = logger.get_labels_for_metric("litellm_deployment_failure_responses") + # Should return some labels (the default ones) + assert isinstance(labels, list) + # Should have more than 0 labels (the default ones) + assert len(labels) > 0 + + +def test_prometheus_metrics_config_type(): + """Test that PrometheusMetricsConfig type validation works""" + # Valid configuration + valid_config = PrometheusMetricsConfig( + group="service_metrics", + metrics=["litellm_deployment_failure_responses"], + include_labels=["litellm_model_name"], + ) + + assert valid_config.group == "service_metrics" + assert valid_config.metrics == ["litellm_deployment_failure_responses"] + assert valid_config.include_labels == ["litellm_model_name"] + + # Test with None include_labels (should be allowed) + config_no_labels = PrometheusMetricsConfig( + group="service_metrics", + metrics=["litellm_deployment_failure_responses"], + include_labels=None, + ) + + assert config_no_labels.include_labels is None + print("PrometheusMetricsConfig type validation passed!") + + +def test_basic_functionality(): + """Test basic functionality without creating multiple instances""" + # Clear registry before test + clear_prometheus_registry() + + # Set up test configuration + test_config = [ + { + "group": "service_metrics", + "metrics": [ + "litellm_deployment_failure_responses", + "litellm_deployment_total_requests", + ], + "include_labels": ["litellm_model_name", "api_provider"], + } + ] + + # Set configuration + litellm.prometheus_metrics_config = test_config + + # Test that the configuration is properly set + assert litellm.prometheus_metrics_config is not None + assert len(litellm.prometheus_metrics_config) == 1 + assert litellm.prometheus_metrics_config[0]["group"] == "service_metrics" + assert ( + "litellm_deployment_failure_responses" + in litellm.prometheus_metrics_config[0]["metrics"] + ) + + print("Basic prometheus configuration test passed!") + + # ============================================================================== # SEMANTIC VALIDATION TESTS - Detect logical errors in metric increments # ==============================================================================