diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index f2d8ec8fb5..722fcd3003 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -51,6 +51,7 @@ from litellm.types.proxy.management_endpoints.model_management_endpoints import UpdateUsefulLinksRequest, ) from litellm.types.router import ( + SPECIAL_MODEL_INFO_PARAMS, Deployment, DeploymentTypedDict, LiteLLMParamsTypedDict, @@ -130,6 +131,32 @@ def update_db_model( updated_patch.model_info.model_dump(exclude_none=True) ) + # Honor explicit-null clears LAST, after both merges, so a model_info blob the UI + # passes through (which today re-sends the OLD pricing on every save) cannot + # silently undo a litellm_params clear via .update(). + # + # Restricted to SPECIAL_MODEL_INFO_PARAMS (input/output cost per token/character + # and cache read/write costs) so this path cannot be used to null out privileged + # model_info fields like team_id or access groups. SPECIAL_MODEL_INFO_PARAMS are + # mirrored between litellm_params and model_info by Deployment.__init__, so the + # clear propagates to both blobs. + if updated_patch.litellm_params: + for field in updated_patch.litellm_params.model_fields_set: + if ( + field in SPECIAL_MODEL_INFO_PARAMS + and getattr(updated_patch.litellm_params, field) is None + ): + merged_deployment_dict["litellm_params"].pop(field, None) # type: ignore + merged_deployment_dict.get("model_info", {}).pop(field, None) + if updated_patch.model_info: + for field in updated_patch.model_info.model_fields_set: + if ( + field in SPECIAL_MODEL_INFO_PARAMS + and getattr(updated_patch.model_info, field) is None + ): + merged_deployment_dict["model_info"].pop(field, None) # type: ignore + merged_deployment_dict.get("litellm_params", {}).pop(field, None) # type: ignore + # convert to prisma compatible format prisma_compatible_model_dict = PrismaCompatibleUpdateDBModel() diff --git a/litellm/types/router.py b/litellm/types/router.py index 6601f552b5..ef7eb05d08 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -398,6 +398,8 @@ SPECIAL_MODEL_INFO_PARAMS = [ "output_cost_per_token", "input_cost_per_character", "output_cost_per_character", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", ] diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index b65f6305b7..85c7c130b3 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -1497,6 +1497,305 @@ class TestUpdateDBModelBlocked: assert "blocked" not in result +def _build_db_model_with_pricing(): + """Wildcard deployment with custom pricing in litellm_params; Deployment.__init__ + mirrors SPECIAL_MODEL_INFO_PARAMS into model_info, so both blobs hold the rate.""" + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + return Deployment( + model_name="openai/*", + litellm_params=LiteLLM_Params( + model="openai/*", + input_cost_per_token=0.000001, + output_cost_per_token=0.000002, + ), + model_info=ModelInfo(id="dep-pricing-0"), + ) + + +class TestUpdateDBModelClearPricing: + """Sending an explicit `null` for a pricing field must remove it from both + `litellm_params` and `model_info` (SPECIAL_MODEL_INFO_PARAMS are mirrored + between the two by Deployment.__init__). + + Restricted to SPECIAL_MODEL_INFO_PARAMS so non-pricing fields (e.g. team_id) + cannot be cleared via this path. + """ + + def test_clear_input_cost_removes_from_both_blobs(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import updateLiteLLMParams + + result = update_db_model( + db_model=_build_db_model_with_pricing(), + updated_patch=updateDeployment( + litellm_params=updateLiteLLMParams(input_cost_per_token=None) + ), + ) + + params = json.loads(result["litellm_params"]) + info = json.loads(result["model_info"]) + assert "input_cost_per_token" not in params + assert "input_cost_per_token" not in info + # Other pricing untouched + assert params.get("output_cost_per_token") == 0.000002 + assert info.get("output_cost_per_token") == 0.000002 + + def test_clear_output_cost_removes_from_both_blobs(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import updateLiteLLMParams + + result = update_db_model( + db_model=_build_db_model_with_pricing(), + updated_patch=updateDeployment( + litellm_params=updateLiteLLMParams(output_cost_per_token=None) + ), + ) + + params = json.loads(result["litellm_params"]) + info = json.loads(result["model_info"]) + assert "output_cost_per_token" not in params + assert "output_cost_per_token" not in info + + def test_non_null_pricing_update_still_works(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import updateLiteLLMParams + + result = update_db_model( + db_model=_build_db_model_with_pricing(), + updated_patch=updateDeployment( + litellm_params=updateLiteLLMParams(input_cost_per_token=0.000005) + ), + ) + + params = json.loads(result["litellm_params"]) + assert params["input_cost_per_token"] == 0.000005 + + def test_omitted_pricing_field_is_preserved(self): + """PATCH semantics: fields not in the patch keep their existing value.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import updateLiteLLMParams + + result = update_db_model( + db_model=_build_db_model_with_pricing(), + updated_patch=updateDeployment( + litellm_params=updateLiteLLMParams(output_cost_per_token=0.000007) + ), + ) + + params = json.loads(result["litellm_params"]) + assert params["input_cost_per_token"] == 0.000001 + assert params["output_cost_per_token"] == 0.000007 + + def test_null_on_non_pricing_field_does_not_clear(self): + """Security guard: only SPECIAL_MODEL_INFO_PARAMS can be cleared via null. + Privileged or unrelated model_info fields (e.g. team_id) must be unaffected + by the null-clearing path so a team admin can't ungate a team-scoped model. + """ + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import ( + Deployment, + LiteLLM_Params, + ModelInfo, + updateLiteLLMParams, + ) + + db_model = Deployment( + model_name="openai/*", + litellm_params=LiteLLM_Params( + model="openai/*", + input_cost_per_token=0.000001, + ), + model_info=ModelInfo(id="dep-pricing-1", team_id="team-keep-me"), + ) + + # Patch sends a null for api_base (non-SPECIAL field). Must NOT clear team_id + # or any other non-pricing field from the merged dict. + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment( + litellm_params=updateLiteLLMParams(api_base=None) + ), + ) + + info = json.loads(result["model_info"]) + # Pricing still present (not part of this patch) + assert "input_cost_per_token" in info + # team_id must survive + assert info.get("team_id") == "team-keep-me" + + def test_clear_survives_model_info_passthrough_with_old_pricing(self): + """Realistic UI submit shape: the patch carries BOTH blobs. The + model_info portion still has the old pricing because the form + re-serializes the source blob. The litellm_params null must beat the + model_info merge — i.e. the clear runs after both merges, not between. + """ + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import ModelInfo, updateLiteLLMParams + + result = update_db_model( + db_model=_build_db_model_with_pricing(), + updated_patch=updateDeployment( + litellm_params=updateLiteLLMParams(input_cost_per_token=None), + # The UI passes the OLD model_info blob through unchanged. + model_info=ModelInfo( + id="dep-pricing-0", + input_cost_per_token=0.000001, # stale value from the page state + ), + ), + ) + + params = json.loads(result["litellm_params"]) + info = json.loads(result["model_info"]) + assert "input_cost_per_token" not in params + assert ( + "input_cost_per_token" not in info + ), "model_info passthrough must not resurrect the cleared override" + + def test_clear_via_model_info_clears_both_blobs(self): + """The mirror works in the reverse direction too: nulling a pricing field + via the model_info patch should clear it from litellm_params as well.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import ModelInfo + + result = update_db_model( + db_model=_build_db_model_with_pricing(), + updated_patch=updateDeployment( + model_info=ModelInfo(id="dep-pricing-0", input_cost_per_token=None) + ), + ) + + params = json.loads(result["litellm_params"]) + info = json.loads(result["model_info"]) + assert "input_cost_per_token" not in params + assert "input_cost_per_token" not in info + + def test_clear_cache_read_cost_removes_from_both_blobs(self): + """cache_read_input_token_cost was added to SPECIAL_MODEL_INFO_PARAMS so + the same null-clear path works for cache-read overrides.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import ( + Deployment, + LiteLLM_Params, + ModelInfo, + updateLiteLLMParams, + ) + + db_model = Deployment( + model_name="openai/*", + litellm_params=LiteLLM_Params( + model="openai/*", + cache_read_input_token_cost=0.0000005, + ), + model_info=ModelInfo(id="dep-cache-read-0"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment( + litellm_params=updateLiteLLMParams(cache_read_input_token_cost=None) + ), + ) + + params = json.loads(result["litellm_params"]) + info = json.loads(result["model_info"]) + assert "cache_read_input_token_cost" not in params + assert "cache_read_input_token_cost" not in info + + def test_clear_cache_write_cost_removes_from_both_blobs(self): + """cache_creation_input_token_cost was added to SPECIAL_MODEL_INFO_PARAMS so + the same null-clear path works for cache-write overrides.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import ( + Deployment, + LiteLLM_Params, + ModelInfo, + updateLiteLLMParams, + ) + + db_model = Deployment( + model_name="openai/*", + litellm_params=LiteLLM_Params( + model="openai/*", + cache_creation_input_token_cost=0.000003, + ), + model_info=ModelInfo(id="dep-cache-write-0"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment( + litellm_params=updateLiteLLMParams(cache_creation_input_token_cost=None) + ), + ) + + params = json.loads(result["litellm_params"]) + info = json.loads(result["model_info"]) + assert "cache_creation_input_token_cost" not in params + assert "cache_creation_input_token_cost" not in info + + def test_clear_cache_read_preserves_other_pricing(self): + """Clearing cache_read must not touch input/output cost overrides.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import ( + Deployment, + LiteLLM_Params, + ModelInfo, + updateLiteLLMParams, + ) + + db_model = Deployment( + model_name="openai/*", + litellm_params=LiteLLM_Params( + model="openai/*", + input_cost_per_token=0.000001, + output_cost_per_token=0.000002, + cache_read_input_token_cost=0.0000005, + cache_creation_input_token_cost=0.000003, + ), + model_info=ModelInfo(id="dep-cache-mixed-0"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment( + litellm_params=updateLiteLLMParams(cache_read_input_token_cost=None) + ), + ) + + params = json.loads(result["litellm_params"]) + info = json.loads(result["model_info"]) + assert "cache_read_input_token_cost" not in params + assert "cache_read_input_token_cost" not in info + # Other pricing untouched in both blobs + assert params["input_cost_per_token"] == 0.000001 + assert params["output_cost_per_token"] == 0.000002 + assert params["cache_creation_input_token_cost"] == 0.000003 + assert info["input_cost_per_token"] == 0.000001 + assert info["output_cost_per_token"] == 0.000002 + assert info["cache_creation_input_token_cost"] == 0.000003 + + class TestGetModelInfoWithIdBlocked: """`ProxyConfig.get_model_info_with_id` must propagate the DB-level `blocked` column into the in-memory `model_info` dict so the router filter can read it.""" diff --git a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/clearCustomPricing.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/clearCustomPricing.spec.ts new file mode 100644 index 0000000000..d21192d237 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/clearCustomPricing.spec.ts @@ -0,0 +1,177 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { Role, users } from "../../fixtures/users"; + +/** + * Regression: clearing the Input / Output / Cache Read / Cache Write Cost + * fields on a deployment with a user-set pricing override must actually remove + * the override from both `litellm_params` and `model_info`. + * + * Pre-fix, the UI sent the old pricing back on every save (the spread of + * `values.litellm_params` re-injected it), and the backend's `exclude_none=True` + * stripped any null that did make it through. End-result: the dashboard + * displayed "Saved" but the override remained in the DB. The cache fields had + * the same bug in a parallel code path and are covered here too. + */ +test.describe("Clear custom pricing on a deployment", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + const masterKey = users[Role.ProxyAdmin].password; + const SEED_INPUT_PER_TOKEN = 0.0000777; + const SEED_OUTPUT_PER_TOKEN = 0.0000999; + const SEED_CACHE_READ_PER_TOKEN = 0.0000333; + const SEED_CACHE_WRITE_PER_TOKEN = 0.0000555; + + // Unique-per-run name so concurrent / repeated runs don't collide on the + // shared dashboard DB. Captured here so afterEach can clean it up. + let createdModelId: string | null = null; + let modelName: string; + + test.beforeEach(async ({ page }) => { + modelName = `e2e-clear-pricing-${Date.now()}`; + const res = await page.request.post("/model/new", { + headers: { Authorization: `Bearer ${masterKey}` }, + data: { + model_name: modelName, + litellm_params: { + model: "openai/gpt-4o", + api_key: "sk-e2e-not-used", + input_cost_per_token: SEED_INPUT_PER_TOKEN, + output_cost_per_token: SEED_OUTPUT_PER_TOKEN, + cache_read_input_token_cost: SEED_CACHE_READ_PER_TOKEN, + cache_creation_input_token_cost: SEED_CACHE_WRITE_PER_TOKEN, + }, + model_info: {}, + }, + }); + expect(res.ok(), `POST /model/new for ${modelName}`).toBe(true); + const body = await res.json(); + createdModelId = body.model_info?.id ?? body.model_id; + expect(createdModelId, "model id from /model/new").toBeTruthy(); + }); + + test.afterEach(async ({ page }) => { + // The dashboard DB persists across this suite (not just per-test), so every + // model created here must be cleaned up regardless of test outcome. + if (createdModelId) { + await page.request.post("/model/delete", { + headers: { Authorization: `Bearer ${masterKey}` }, + data: { id: createdModelId }, + }); + createdModelId = null; + } + }); + + test("UI sends null for cleared pricing and backend removes the override", async ({ + page, + }) => { + // Navigate to the model detail view. + await page.goto("/ui"); + await page.getByText("Models + Endpoints").click(); + + const modelRow = page.locator("tr", { hasText: modelName }).first(); + await expect(modelRow).toBeVisible({ timeout: 15_000 }); + await modelRow.click(); + await expect(page.getByText("Back to Models").first()).toBeVisible({ + timeout: 10_000, + }); + + // Sanity: the seeded pricing is shown in the detail view (77.7000 / 99.9000 + // per 1M tokens). The dashboard renders the per-token rate × 1e6. + await expect(page.getByText("77.7000")).toBeVisible({ timeout: 10_000 }); + await expect(page.getByText("99.9000")).toBeVisible({ timeout: 10_000 }); + + // Open the edit form and clear all four pricing fields. + await page.getByRole("button", { name: "Edit Settings" }).click(); + const inputCost = page.getByPlaceholder("Enter input cost"); + const outputCost = page.getByPlaceholder("Enter output cost"); + // Both cache fields share the same placeholder ("Defaults to Input Cost if blank"), + // so disambiguate via the Form.Item id (AntD assigns the `name` prop as input id). + const cacheReadCost = page.locator("#cache_read_cost"); + const cacheWriteCost = page.locator("#cache_write_cost"); + await inputCost.waitFor({ timeout: 15_000 }); + for (const field of [inputCost, outputCost, cacheReadCost, cacheWriteCost]) { + await field.click({ clickCount: 3 }); + await page.keyboard.press("Delete"); + } + + // Capture the outgoing PATCH so we can assert the UI sends explicit nulls. + const patchPromise = page.waitForRequest( + (req) => + req.method() === "PATCH" && + req.url().includes(`/model/${createdModelId}/update`) + ); + await page.getByRole("button", { name: "Save Changes" }).click(); + const patchReq = await patchPromise; + const patchBody = JSON.parse(patchReq.postData() ?? "{}"); + expect( + patchBody.litellm_params.input_cost_per_token, + "UI sends explicit null for cleared input cost" + ).toBeNull(); + expect( + patchBody.litellm_params.output_cost_per_token, + "UI sends explicit null for cleared output cost" + ).toBeNull(); + expect( + patchBody.litellm_params.cache_read_input_token_cost, + "UI sends explicit null for cleared cache_read cost" + ).toBeNull(); + expect( + patchBody.litellm_params.cache_creation_input_token_cost, + "UI sends explicit null for cleared cache_write cost" + ).toBeNull(); + + // Success toast confirms the save was accepted. + await expect( + page.getByText("Model settings updated successfully") + ).toBeVisible({ timeout: 10_000 }); + + // Verify via the management API: the user-set rate is gone from both blobs. + // The cost-map may synthesize a default for known providers in the response, + // so the assertion is "no longer the seeded value" rather than literally + // undefined. + const infoRes = await page.request.get( + `/v2/model/info?include_team_models=true&page=1&size=100&modelId=${createdModelId}`, + { headers: { Authorization: `Bearer ${masterKey}` } } + ); + expect(infoRes.ok()).toBe(true); + const infoBody = await infoRes.json(); + const row = (infoBody.data ?? infoBody).find?.( + (m: any) => m?.model_info?.id === createdModelId + ); + expect(row, "model info row").toBeTruthy(); + + expect( + "input_cost_per_token" in row.litellm_params, + "litellm_params.input_cost_per_token key removed" + ).toBe(false); + expect( + "output_cost_per_token" in row.litellm_params, + "litellm_params.output_cost_per_token key removed" + ).toBe(false); + expect( + "cache_read_input_token_cost" in row.litellm_params, + "litellm_params.cache_read_input_token_cost key removed" + ).toBe(false); + expect( + "cache_creation_input_token_cost" in row.litellm_params, + "litellm_params.cache_creation_input_token_cost key removed" + ).toBe(false); + expect( + row.model_info.input_cost_per_token, + "model_info.input_cost_per_token no longer the seeded override" + ).not.toBe(SEED_INPUT_PER_TOKEN); + expect( + row.model_info.output_cost_per_token, + "model_info.output_cost_per_token no longer the seeded override" + ).not.toBe(SEED_OUTPUT_PER_TOKEN); + expect( + row.model_info.cache_read_input_token_cost, + "model_info.cache_read_input_token_cost no longer the seeded override" + ).not.toBe(SEED_CACHE_READ_PER_TOKEN); + expect( + row.model_info.cache_creation_input_token_cost, + "model_info.cache_creation_input_token_cost no longer the seeded override" + ).not.toBe(SEED_CACHE_WRITE_PER_TOKEN); + }); +}); diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index 5ed4c0468b..768083be6e 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -256,14 +256,26 @@ export default function ModelInfoView({ tags: values.tags, }; - if (form.isFieldTouched("input_cost") && values.input_cost !== undefined && values.input_cost !== null) { - updatedLitellmParams.input_cost_per_token = Number(values.input_cost) / 1_000_000; + if (form.isFieldTouched("input_cost")) { + if (values.input_cost !== undefined && values.input_cost !== null && values.input_cost !== "") { + updatedLitellmParams.input_cost_per_token = Number(values.input_cost) / 1_000_000; + } else { + // Explicit null signals the backend to remove the pricing override. + updatedLitellmParams.input_cost_per_token = null; + } } - if (form.isFieldTouched("output_cost") && values.output_cost !== undefined && values.output_cost !== null) { - updatedLitellmParams.output_cost_per_token = Number(values.output_cost) / 1_000_000; + if (form.isFieldTouched("output_cost")) { + if (values.output_cost !== undefined && values.output_cost !== null && values.output_cost !== "") { + updatedLitellmParams.output_cost_per_token = Number(values.output_cost) / 1_000_000; + } else { + updatedLitellmParams.output_cost_per_token = null; + } } - // Cache Read Cost: explicit value if provided, else fall back to input cost (when input cost touched). + // Cache Read Cost: + // - explicit value provided → use it + // - field touched but empty → explicit null (signals backend to remove override) + // - only input_cost touched → fall back to input_cost (guarded against null) if (form.isFieldTouched("cache_read_cost") || form.isFieldTouched("input_cost")) { if ( values.cache_read_cost !== undefined && @@ -271,14 +283,19 @@ export default function ModelInfoView({ values.cache_read_cost !== "" ) { updatedLitellmParams.cache_read_input_token_cost = Number(values.cache_read_cost) / 1_000_000; - } else if (updatedLitellmParams.input_cost_per_token !== undefined) { + } else if (form.isFieldTouched("cache_read_cost")) { + updatedLitellmParams.cache_read_input_token_cost = null; + } else if ( + updatedLitellmParams.input_cost_per_token !== undefined && + updatedLitellmParams.input_cost_per_token !== null + ) { updatedLitellmParams.cache_read_input_token_cost = updatedLitellmParams.input_cost_per_token; } } - // Cache Write Cost: explicit value if provided, else clear the override - // so the backend falls back to the model-level default. Sending 0 here - // would persist a zero rate even when the user intended to unset it. + // Cache Write Cost: explicit value if provided, else explicit null so the + // backend removes the override and falls back to the model-level default. + // Sending 0 here would persist a zero rate even when the user intended to unset it. if (form.isFieldTouched("cache_write_cost")) { if ( values.cache_write_cost !== undefined && @@ -287,7 +304,7 @@ export default function ModelInfoView({ ) { updatedLitellmParams.cache_creation_input_token_cost = Number(values.cache_write_cost) / 1_000_000; } else { - delete updatedLitellmParams.cache_creation_input_token_cost; + updatedLitellmParams.cache_creation_input_token_cost = null; } }