From b0ea013042d5fb58cd4f5254f701c4465e6854ba Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 27 May 2026 15:52:19 -0700 Subject: [PATCH] test(e2e): cover add-fallback flow in Router Settings (#29069) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(e2e): cover add-fallback flow in Router Settings as proxy admin The Router Settings → Fallbacks → Add Fallbacks flow was an uncovered manual-QA path. This adds a test that opens the modal, picks a primary + fallback from the seeded mock models, saves, and verifies both render in the fallback table. * fix(e2e): make router-fallback test idempotent and pick antd options by text - Match `.ant-select-item-option` by text instead of `getByTitle(...)` — FallbackGroupConfig uses `options=` (not children), so no `title` attribute is emitted and the title-based selector hangs. - Add before/after hooks that wipe any fallback for fake-openai-gpt-4 via /config/update so retries and local reruns don't trip on leftover state. - Tighten the success assertion to a single tbody row containing BOTH the primary and the fallback names — pre-existing rows can no longer vacuously satisfy the check. - Fix the stale "Three tabs" comment to "Four tabs". Addresses Greptile P2s on PR #29069. * fix(e2e): keyboard-select fallback models + correct cleanup endpoint - Replace mouse-based option clicks with click-to-focus + type + Enter. FallbackGroupConfig's Selects use `options=` and a custom getPopupContainer, so locating options via `.ant-select-dropdown` hit several races: DOM-clicks left antd's popup state stale (the primary popup then intercepted the fallback click), `getByRole` matched always-mounted hidden options, and pointer stability fought the open animation. Typing into the showSearch input narrows the listbox to one option and Enter selects it cleanly. - Assert on dialog-side state changes (the active tab adopts the primary model name; the chain helper shows "1/10 used") instead of popup contents — these reflect the actual selection landing. - Cleanup helper now hits /get/config/callbacks (the real endpoint; /get/callbacks returns 404), so the before/after reset actually clears prior router_settings.fallbacks state. --- .../tests/settings/routerSettings.spec.ts | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts diff --git a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts new file mode 100644 index 0000000000..8dd5571f7a --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts @@ -0,0 +1,102 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { navigateToPage } from "../../helpers/navigation"; +import { Page } from "../../fixtures/pages"; +import { Role, users } from "../../fixtures/users"; + +const PRIMARY = "fake-openai-gpt-4"; +const FALLBACK = "fake-anthropic-claude"; + +/** + * Wipe any fallbacks for the primary model so the test is idempotent across + * retries and local reruns (the proxy persists router_settings to the DB). + */ +async function clearFallbackForPrimary(request: import("@playwright/test").APIRequestContext) { + const masterKey = users[Role.ProxyAdmin].password; + const auth = { Authorization: `Bearer ${masterKey}` }; + + const current = await request.get("http://localhost:4000/get/config/callbacks", { headers: auth }); + if (!current.ok()) return; + const body = await current.json(); + const router = body?.router_settings ?? {}; + const existing: Array> = Array.isArray(router.fallbacks) ? router.fallbacks : []; + const next = existing.filter((entry) => !(entry && PRIMARY in entry)); + if (next.length === existing.length) return; + + await request.post("http://localhost:4000/config/update", { + headers: auth, + data: { router_settings: { ...router, fallbacks: next } }, + }); +} + +test.describe("Router Settings - Fallbacks", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test.beforeEach(async ({ request }) => { + await clearFallbackForPrimary(request); + }); + + test.afterEach(async ({ request }) => { + await clearFallbackForPrimary(request); + }); + + test("Add a fallback and verify it appears in the table", async ({ page }) => { + await navigateToPage(page, Page.RouterSettings); + + // Four tabs: Loadbalancing / Routing Groups / Fallbacks / General — click Fallbacks + await page.getByRole("tab", { name: "Fallbacks" }).click(); + + // The model options come from /model_group/info, which AddFallbacks + // fires only after the modal mounts. Wait for that response so the + // dropdown is populated before we try to pick from it — without this + // the test races on CI (local SLOWMO masks the gap). + const modelsLoaded = page.waitForResponse( + (res) => res.url().includes("/model_group/info") && res.status() === 200, + { timeout: 15_000 }, + ); + await page.getByRole("button", { name: /Add Fallbacks/i }).click(); + await modelsLoaded; + + const modal = page.locator(".ant-modal:visible"); + await expect(modal).toBeVisible({ timeout: 5_000 }); + + // FallbackGroupConfig.tsx renders both selects with `showSearch`. The + // most stable interaction is: click to open + focus, type the model name to + // narrow the listbox to a single highlighted option, then press Enter. + // Verify each selection landed by watching the dialog's own state transition + // (the tab title updates to the picked primary; the fallback chain list + // populates) rather than by asserting on the dropdown popup, which sits in + // a custom getPopupContainer and is awkward to scope reliably. + const primarySelect = modal.locator(".ant-select").filter({ hasText: "Select primary model" }); + await primarySelect.click(); + await page.keyboard.type(PRIMARY); + await page.keyboard.press("Enter"); + await expect(modal.getByRole("tab", { name: PRIMARY })).toBeVisible({ timeout: 10_000 }); + + const fallbackSelect = modal.locator(".ant-select").filter({ hasText: "Select fallback models" }); + await fallbackSelect.click(); + await page.keyboard.type(FALLBACK); + await page.keyboard.press("Enter"); + await page.keyboard.press("Escape"); + // The Fallback Chain helper text reads "(N/10 used)"; once it ticks to 1 the + // selection has been recorded. + await expect(modal.getByText("(1/10 used)")).toBeVisible({ timeout: 10_000 }); + + // Save + await modal.getByRole("button", { name: /Save All Configurations/i }).click(); + + // Success toast + await expect(page.getByText(/fallback configuration\(s\) added successfully/i).first()) + .toBeVisible({ timeout: 10_000 }); + + // Modal closes, and a single row contains BOTH the primary and the fallback + // model — stronger than asserting each name appears somewhere in tbody, + // which could be satisfied by leftover rows from prior runs. + await expect(modal).not.toBeVisible({ timeout: 5_000 }); + + const newRow = page.locator("table tbody tr") + .filter({ hasText: PRIMARY }) + .filter({ hasText: FALLBACK }); + await expect(newRow).toHaveCount(1, { timeout: 10_000 }); + }); +});