Request Headers - support x-litellm-num-retries + Usage - support usage by model group (BerriAI#12890)

* feat(litellm_pre_call_utils.py): add num_retries to litellm data for backend call

allow user to pass in num retries via request headers

* test(test_litellm_pre_call_utils.py): add unit test

* docs(request_headers.md): document new request header

* fix(common_daily_activity.py): show spend breakdown by model group

Partial fix for BerriAI#12887

* feat(new_usage.tsx): new tab switcher for viewing usage by model group vs. received model

Closes BerriAI#12887

Author: krrishdholakia

docs/my-website/docs/proxy/request_headers.md

@@ -10,6 +10,8 @@ Special headers that are supported by LiteLLM.
 
 `x-litellm-tags`: Optional[str]: A comma separated list (e.g. `tag1,tag2,tag3`) of tags to use for [tag-based routing](./tag_routing) **OR** [spend-tracking](./enterprise.md#tracking-spend-for-custom-tags).
 
+`x-litellm-num-retries`: Optional[int]: The number of retries for the request.
+
 ## Anthropic Headers
 
 `anthropic-version` Optional[str]: The version of the Anthropic API to use.  

litellm/proxy/_types.py

@@ -2780,6 +2780,7 @@ class LitellmDataForBackendLLMCall(TypedDict, total=False):
     organization: str
     timeout: Optional[float]
     user: Optional[str]
+    num_retries: Optional[int]
 
 
 class JWTKeyItem(TypedDict, total=False):

litellm/proxy/litellm_pre_call_utils.py

@@ -272,6 +272,16 @@ def _get_timeout_from_request(headers: dict) -> Optional[float]:
             return float(timeout_header)
         return None
 
+    @staticmethod
+    def _get_num_retries_from_request(headers: dict) -> Optional[int]:
+        """
+        Workaround for client request from Vercel's AI SDK.
+        """
+        num_retries_header = headers.get("x-litellm-num-retries", None)
+        if num_retries_header is not None:
+            return int(num_retries_header)
+        return None
+
     @staticmethod
     def _get_forwardable_headers(
         headers: Union[Headers, dict],
@@ -407,6 +417,10 @@ def add_litellm_data_for_backend_llm_call(
         if timeout is not None:
             data["timeout"] = timeout
 
+        num_retries = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers)
+        if num_retries is not None:
+            data["num_retries"] = num_retries
+
         return data
 
     @staticmethod
@@ -801,7 +815,10 @@ async def add_litellm_data_to_request(  # noqa: PLR0915
                 data[k] = v
 
     # Add disabled callbacks from key metadata
-    if user_api_key_dict.metadata and "litellm_disabled_callbacks" in user_api_key_dict.metadata:
+    if (
+        user_api_key_dict.metadata
+        and "litellm_disabled_callbacks" in user_api_key_dict.metadata
+    ):
         disabled_callbacks = user_api_key_dict.metadata["litellm_disabled_callbacks"]
         if disabled_callbacks and isinstance(disabled_callbacks, list):
             data["litellm_disabled_callbacks"] = disabled_callbacks

litellm/proxy/management_endpoints/common_daily_activity.py

@@ -80,6 +80,44 @@ def update_breakdown_metrics(
             )
         )
 
+    # Update model group breakdown
+    if record.model_group and record.model_group not in breakdown.model_groups:
+        breakdown.model_groups[record.model_group] = MetricWithMetadata(
+            metrics=SpendMetrics(),
+            metadata=model_metadata.get(record.model_group, {}),
+        )
+    if record.model_group:
+        breakdown.model_groups[record.model_group].metrics = update_metrics(
+            breakdown.model_groups[record.model_group].metrics, record
+        )
+
+        # Update API key breakdown for this model
+        if (
+            record.api_key
+            not in breakdown.model_groups[record.model_group].api_key_breakdown
+        ):
+            breakdown.model_groups[record.model_group].api_key_breakdown[
+                record.api_key
+            ] = KeyMetricWithMetadata(
+                metrics=SpendMetrics(),
+                metadata=KeyMetadata(
+                    key_alias=api_key_metadata.get(record.api_key, {}).get(
+                        "key_alias", None
+                    ),
+                    team_id=api_key_metadata.get(record.api_key, {}).get(
+                        "team_id", None
+                    ),
+                ),
+            )
+        breakdown.model_groups[record.model_group].api_key_breakdown[
+            record.api_key
+        ].metrics = update_metrics(
+            breakdown.model_groups[record.model_group]
+            .api_key_breakdown[record.api_key]
+            .metrics,
+            record,
+        )
+
     if record.mcp_namespaced_tool_name:
         if record.mcp_namespaced_tool_name not in breakdown.mcp_servers:
             breakdown.mcp_servers[record.mcp_namespaced_tool_name] = MetricWithMetadata(
@@ -295,22 +333,6 @@ async def get_daily_activity(
             take=page_size,
         )
 
-        # # for 50% of the records, set the mcp_server_id to a random value
-        # mcp_server_dict = {"Zapier_Gmail_MCP", "Stripe_MCP"}
-        # import random
-
-        # for idx, record in enumerate(daily_spend_data):
-        #     record = LiteLLM_DailyUserSpend(**record.model_dump())
-        #     if random.random() < 0.5:
-        #         record.mcp_server_id = random.choice(list(mcp_server_dict))
-        #         record.model = None
-        #         record.model_group = None
-        #         record.prompt_tokens = 0
-        #         record.completion_tokens = 0
-        #         record.cache_read_input_tokens = 0
-        #         record.cache_creation_input_tokens = 0
-        #     daily_spend_data[idx] = record
-
         # Get all unique API keys from the spend data
         api_keys = set()
         for record in daily_spend_data:

litellm/types/proxy/management_endpoints/common_daily_activity.py

@@ -61,6 +61,9 @@ class BreakdownMetrics(BaseModel):
     models: Dict[str, MetricWithMetadata] = Field(
         default_factory=dict
     )  # model -> {metrics, metadata}
+    model_groups: Dict[str, MetricWithMetadata] = Field(
+        default_factory=dict
+    )  # model_group -> {metrics, metadata}
     providers: Dict[str, MetricWithMetadata] = Field(
         default_factory=dict
     )  # provider -> {metrics, metadata}