test(ui): data-driven App Router migration E2E smoke (default + server-root-path) (#29974)

* test(ui): add a data-driven App Router migration E2E smoke

Add a growing Playwright smoke for migrated pages: for each segment it deep-links
to the path route, asserts the URL and that the dashboard shell rendered, then
clicks off to a legacy page and asserts navigation still works. Driven by
e2e_tests/fixtures/migratedPages.ts, so adding a page is one line.

Runs in two situations against the same proxy: the default mount (npm run
e2e:migration) and a non-root SERVER_ROOT_PATH mount (npm run e2e:migration:root).
globalSetup now logs in at `${SERVER_ROOT_PATH}/ui/login` so the admin storage
state is valid under a prefix. Seeded with api-reference; append the rest as their
migrations merge.

* test(ui): support headed slow-motion + watch pauses in the migration smoke

Honor SLOWMO in the server-root-path config (the default config already did),
and add an env-gated E2E_WATCH_MS pause so a headed run lingers on each state.
Both are no-ops by default, so CI behavior is unchanged.

* test(ui): make the migration smoke a sidebar-click user journey

Rework the smoke from deep-linking to a real navigation journey: start at the
landing page, click the migrated page in the sidebar (expanding submenus for
nested items), assert the path route rendered, reload it (the check a wrong
server_root_path breaks), bounce to a legacy page and back, and — once two pages
are migrated — navigate directly between two migrated pages. Verifies via URL +
shell render, driven by the same fixture list.

* test(ui): address review on the migration smoke

Escape ROOT and segment before interpolating them into RegExp URL matchers so a
future segment containing regex metacharacters can't silently widen the match.
Make the server-root-path config fail fast when SERVER_ROOT_PATH is unset instead
of silently re-running the default mount and passing without exercising the prefix.

* test(ui): drop unused watch helper and fix stale smoke README

* test(ui): run the migration smoke under a server root path in CI

* test(ui): harden + instrument the server-root-path proxy reboot in CI

* test(ui): run the server-root-path migration smoke as its own CI job

Replace the in-place proxy reboot in e2e_ui_testing with a dedicated
e2e_ui_testing_server_root_path job that boots the proxy once with
SERVER_ROOT_PATH=/litellm, matching how every other proxy variant in the
config gets its own job rather than killing and relaunching the live proxy.

The reboot was failing deterministically: after pkill -9 and relaunch the
prefixed proxy never came back up on :4000 (connection refused), so the smoke
never ran. The readiness step that was supposed to surface the cause could
never reach its boot-log tail because CircleCI runs steps under bash -eo
pipefail and the preceding `curl -sv ... | tail` aborted the step with curl's
exit 7. Booting the proxy as the job's own background step lets any boot crash
land in that step's log instead of being swallowed.

The default e2e_ui_testing job is unchanged aside from dropping the reboot,
prefixed-readiness, and prefixed-smoke steps; the migration smoke still runs at
the root mount there via the default Playwright config.

.circleci/config.yml

@@ -2690,6 +2690,122 @@ jobs:
           path: ui/litellm-dashboard/playwright-report
           destination: e2e-playwright-report
 
+  e2e_ui_testing_server_root_path:
+    docker:
+      - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
+        environment:
+          POSTGRES_USER: e2euser
+          POSTGRES_PASSWORD: e2epassword
+          POSTGRES_DB: litellm_e2e
+    resource_class: large
+    working_directory: ~/project
+    environment:
+      DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
+      CI: "true"
+      # The whole job exercises the proxy mounted under a prefix. SERVER_ROOT_PATH
+      # is read both by the proxy at boot (to rewrite the built UI bundle in place)
+      # and by migration.serverRootPath.config.ts, which refuses to run without it.
+      SERVER_ROOT_PATH: "/litellm"
+    steps:
+      - checkout
+      - setup_google_dns
+      - install_uv
+      - restore_cache:
+          keys:
+            - v1-uv-cache-{{ checksum "uv.lock" }}
+      - run:
+          name: Install Python dependencies
+          command: |
+            uv sync --frozen --all-groups --all-extras --python 3.12
+            uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma
+      - save_cache:
+          key: v1-uv-cache-{{ checksum "uv.lock" }}
+          paths:
+            - ~/.cache/uv
+      - restore_cache:
+          keys:
+            - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+      - run:
+          name: Install Node dependencies and Playwright
+          command: |
+            cd ui/litellm-dashboard
+            npm ci
+            npx playwright install chromium
+      - save_cache:
+          key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+          paths:
+            - ui/litellm-dashboard/node_modules
+            - ~/.cache/ms-playwright
+      - run:
+          name: Build UI from source
+          command: |
+            cd ui/litellm-dashboard
+            npm run build
+            rm -rf ../../litellm/proxy/_experimental/out
+            mv out ../../litellm/proxy/_experimental/out
+            find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do
+              d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html"
+            done
+      - wait_for_service:
+          url: tcp://localhost:5432
+          timeout: "30"
+      - run:
+          name: Push Prisma schema
+          command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
+      - run:
+          name: Seed database
+          command: |
+            PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
+              -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
+      - run:
+          name: Start mock LLM server
+          command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
+          background: true
+      - run:
+          name: Start LiteLLM proxy under a server root path
+          environment:
+            LITELLM_MASTER_KEY: "sk-1234"
+            MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
+            DISABLE_SCHEMA_UPDATE: "true"
+          # Output flows to this step's own log, so a boot crash is visible here
+          # rather than swallowed by a downstream readiness probe.
+          command: |
+            LITELLM_LICENSE="$LITELLM_LICENSE" \
+              uv run --no-sync python -m litellm.proxy.proxy_cli \
+                --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
+                --port 4000
+          background: true
+      - run:
+          name: Wait for prefixed proxy to be ready
+          command: |
+            for i in $(seq 1 60); do
+              HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer sk-1234" http://127.0.0.1:4000/litellm/health 2>/dev/null || true)
+              if [ "$HTTP_CODE" = "200" ]; then
+                echo "Prefixed proxy is ready"
+                exit 0
+              fi
+              sleep 2
+            done
+            echo "Prefixed proxy failed to start; see the 'Start LiteLLM proxy under a server root path' step for the boot log"
+            exit 1
+      - run:
+          name: Run migration smoke under SERVER_ROOT_PATH
+          command: |
+            cd ui/litellm-dashboard
+            LITELLM_LICENSE="$LITELLM_LICENSE" \
+              npx playwright test --config e2e_tests/migration.serverRootPath.config.ts
+          no_output_timeout: 10m
+      - store_artifacts:
+          path: ui/litellm-dashboard/test-results
+          destination: e2e-server-root-path-test-results
+      - store_artifacts:
+          path: ui/litellm-dashboard/playwright-report
+          destination: e2e-server-root-path-playwright-report
+
   build_docker_database_image:
     machine:
       image: ubuntu-2204:2024.04.1
@@ -2795,6 +2911,8 @@ workflows:
           filters: *main_branches
       - e2e_ui_testing:
           filters: *main_branches
+      - e2e_ui_testing_server_root_path:
+          filters: *main_branches
       - build_and_test:
           requires:
             - build_docker_database_image

ui/litellm-dashboard/e2e_tests/fixtures/migratedPages.ts

@@ -0,0 +1,16 @@
+/**
+ * Source of truth for the App Router migration smoke (tests/migration/migratedPages.spec.ts).
+ *
+ * Add a route segment here once its migration has MERGED to the branch under test.
+ * Both suites pick it up automatically:
+ *   - default mount:           npm run e2e:migration
+ *   - server-root-path mount:  SERVER_ROOT_PATH=/<root> npm run e2e:migration:root
+ *
+ * Keep this in lockstep with MIGRATED_PAGES in src/utils/migratedPages.ts.
+ * Pending (uncomment as each PR lands): playground, and the leaf-pages batch
+ * (budgets, caching, cost-tracking, guardrails, guardrails-monitor, logs,
+ * mcp-servers, memory, policies, projects, prompts, search-tools, skills,
+ * tag-management, tool-policies, transform-request, ui-theme, vector-stores,
+ * workflows, access-groups).
+ */
+export const MIGRATED_E2E_SEGMENTS: string[] = ["api-reference"];

ui/litellm-dashboard/e2e_tests/globalSetup.ts

@@ -4,17 +4,18 @@ import * as fs from "fs";
 
 async function globalSetup() {
   const browser = await chromium.launch();
+  const rootPath = process.env.SERVER_ROOT_PATH ?? "";
 
   for (const role of Object.values(Role)) {
     const { email, password } = users[role];
     const storagePath = STORAGE_PATHS[role];
     const page = await browser.newPage();
     try {
-      await page.goto("http://localhost:4000/ui/login");
+      await page.goto(`http://localhost:4000${rootPath}/ui/login`);
       await page.getByPlaceholder("Enter your username").fill(email);
       await page.getByPlaceholder("Enter your password").fill(password);
       await page.getByRole("button", { name: "Login", exact: true }).click();
-      await page.waitForURL((url) => url.pathname.startsWith("/ui") && !url.pathname.includes("/login"), {
+      await page.waitForURL((url) => url.pathname.startsWith(`${rootPath}/ui`) && !url.pathname.includes("/login"), {
         timeout: 30_000,
       });
       await expect(page.locator("a", { hasText: "Virtual Keys" })).toBeVisible({ timeout: 30_000 });

ui/litellm-dashboard/e2e_tests/migration.serverRootPath.config.ts

@@ -0,0 +1,38 @@
+import { defineConfig, devices } from "@playwright/test";
+
+/**
+ * App Router migration smoke under a non-root mount. Boot the proxy with the same
+ * SERVER_ROOT_PATH (e.g. SERVER_ROOT_PATH=/litellm) and a UI built for it before
+ * running. globalSetup logs in at `${SERVER_ROOT_PATH}/ui/login` so the admin
+ * storage state is valid under the prefix.
+ */
+if (!process.env.SERVER_ROOT_PATH) {
+  throw new Error(
+    "migration.serverRootPath.config.ts requires SERVER_ROOT_PATH to be set (e.g. SERVER_ROOT_PATH=/litellm). " +
+      "Without it this config silently re-runs the default mount and never exercises the prefix. " +
+      "For the root-less run use the default playwright.config.ts (npm run e2e:migration).",
+  );
+}
+
+export default defineConfig({
+  testDir: "./tests/migration",
+  testMatch: ["migratedPages.spec.ts"],
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  workers: process.env.CI ? 1 : undefined,
+  reporter: "list",
+  use: {
+    baseURL: "http://localhost:4000",
+    trace: "on-first-retry",
+    actionTimeout: 15 * 1000,
+    navigationTimeout: 30 * 1000,
+    launchOptions: {
+      slowMo: process.env.SLOWMO ? parseInt(process.env.SLOWMO, 10) || 0 : 0,
+    },
+  },
+  projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }],
+  timeout: 3 * 60 * 1000,
+  expect: { timeout: 10 * 1000 },
+  globalSetup: require.resolve("./globalSetup"),
+});

ui/litellm-dashboard/e2e_tests/tests/migration/README.md

@@ -0,0 +1,33 @@
+# App Router migration smoke
+
+A growing E2E smoke for pages migrated from the legacy `?page=` switch to App
+Router path routes. For each migrated page it clicks the page's sidebar link, checks
+the URL is the path route and the page renders, reloads it, then clicks off to a
+legacy page and back to confirm navigation still works. It runs in two situations:
+the default mount and a non-root `SERVER_ROOT_PATH` mount.
+
+## Adding a page
+
+When a page's migration merges, add its route segment to
+`e2e_tests/fixtures/migratedPages.ts` (keep it in lockstep with `MIGRATED_PAGES`
+in `src/utils/migratedPages.ts`). Both suites pick it up automatically.
+
+## Running
+
+Build the UI into the proxy and start the proxy first (the suite runs against
+`http://localhost:4000`).
+
+Default mount:
+
+```
+npm run e2e:migration
+```
+
+Non-root mount (build and boot the proxy with the same root path, e.g. `/litellm`):
+
+```
+SERVER_ROOT_PATH=/litellm npm run e2e:migration:root
+```
+
+`globalSetup` logs in once per role; the admin storage state is reused for these
+tests. Under a non-root mount it logs in at `${SERVER_ROOT_PATH}/ui/login`.

ui/litellm-dashboard/e2e_tests/tests/migration/migratedPages.spec.ts

@@ -0,0 +1,101 @@
+import { test, expect, type Page } from "@playwright/test";
+import { MIGRATED_E2E_SEGMENTS } from "../../fixtures/migratedPages";
+import { ADMIN_STORAGE_PATH } from "../../constants";
+import { dismissFeedbackPopup } from "../../helpers/navigation";
+
+/**
+ * App Router migration smoke as a user journey: start where the proxy lands you,
+ * click a migrated page in the sidebar, confirm it routed and rendered, reload it
+ * (the check a wrong server_root_path breaks), bounce to a legacy page and back,
+ * and, once two pages are migrated, navigate directly between two migrated pages.
+ *
+ * Driven by MIGRATED_E2E_SEGMENTS, so it grows as pages are migrated. Set
+ * SERVER_ROOT_PATH (e.g. "/litellm") to exercise the non-root mount; leave it
+ * unset for the default mount. Boot the proxy with the matching value first.
+ */
+const ROOT = process.env.SERVER_ROOT_PATH ?? "";
+
+const esc = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+const pathRe = (segment: string) => new RegExp(`${esc(ROOT)}/ui/${esc(segment)}/?($|\\?)`);
+const legacyAnchor = (page: Page) => page.locator("a", { hasText: "Virtual Keys" });
+
+/** The dashboard shell is present (sidebar rendered); page didn't 404 / crash. */
+async function expectRendered(page: Page) {
+  await expect(legacyAnchor(page)).toBeVisible({ timeout: 20_000 });
+}
+
+/**
+ * Click a migrated page's sidebar link. Migrated items render as <a href=".../ui/<segment>">;
+ * nested ones live under collapsible submenus, so expand submenus until the link is clickable.
+ */
+async function clickSidebar(page: Page, segment: string) {
+  const link = page.locator(`a[href$="/ui/${segment}"]`).first();
+  for (let i = 0; i < 8 && !(await link.isVisible().catch(() => false)); i++) {
+    const collapsedSubmenu = page
+      .locator(".ant-menu-submenu:not(.ant-menu-submenu-open) > .ant-menu-submenu-title")
+      .first();
+    if (!(await collapsedSubmenu.isVisible().catch(() => false))) break;
+    await collapsedSubmenu.click();
+    await page.waitForTimeout(250);
+  }
+  await link.click();
+}
+
+test.use({ storageState: ADMIN_STORAGE_PATH });
+
+test.describe("App Router migrated pages", () => {
+  for (const segment of MIGRATED_E2E_SEGMENTS) {
+    test(`${segment}: sidebar nav, reload, and round-trip with a legacy page`, async ({ page }) => {
+      const pageErrors: string[] = [];
+      page.on("pageerror", (e) => pageErrors.push(String(e)));
+
+      // 1. Start where the proxy lands us.
+      await page.goto(`${ROOT}/ui/`);
+      await dismissFeedbackPopup(page);
+      await expectRendered(page);
+
+      // 2. Click the migrated page in the sidebar -> path route + rendered.
+      await clickSidebar(page, segment);
+      await expect(page).toHaveURL(pathRe(segment));
+      await expectRendered(page);
+      // 3. Reload the path route directly; a wrong server_root_path 404s here.
+      await page.reload();
+      await dismissFeedbackPopup(page);
+      await expect(page).toHaveURL(pathRe(segment));
+      await expectRendered(page);
+      // 4. Click off to a legacy (not-yet-migrated) page.
+      await legacyAnchor(page).click();
+      await expect(page).toHaveURL(new RegExp(`${esc(ROOT)}/ui/\\?page=api-keys`));
+      await dismissFeedbackPopup(page);
+      await expectRendered(page);
+      // 5. Click back to the migrated page.
+      await clickSidebar(page, segment);
+      await expect(page).toHaveURL(pathRe(segment));
+      await expectRendered(page);
+      expect(pageErrors, `page errors during ${segment} journey`).toEqual([]);
+    });
+  }
+
+  test("navigates directly between two migrated pages", async ({ page }) => {
+    test.skip(MIGRATED_E2E_SEGMENTS.length < 2, "needs >= 2 migrated pages");
+    const [first, second] = MIGRATED_E2E_SEGMENTS;
+    const pageErrors: string[] = [];
+    page.on("pageerror", (e) => pageErrors.push(String(e)));
+
+    await page.goto(`${ROOT}/ui/`);
+    await dismissFeedbackPopup(page);
+
+    await clickSidebar(page, first);
+    await expect(page).toHaveURL(pathRe(first));
+    await expectRendered(page);
+    await clickSidebar(page, second);
+    await expect(page).toHaveURL(pathRe(second));
+    await expectRendered(page);
+    // Back to the first migrated page.
+    await clickSidebar(page, first);
+    await expect(page).toHaveURL(pathRe(first));
+    await expectRendered(page);
+
+    expect(pageErrors, "page errors during migrated -> migrated nav").toEqual([]);
+  });
+});

ui/litellm-dashboard/package.json

@@ -16,6 +16,8 @@
     "format:check": "prettier --check .",
     "e2e": "playwright test --config e2e_tests/playwright.config.ts",
     "e2e:ui": "playwright test --ui --config e2e_tests/playwright.config.ts",
+    "e2e:migration": "playwright test e2e_tests/tests/migration/migratedPages.spec.ts --config e2e_tests/playwright.config.ts",
+    "e2e:migration:root": "playwright test --config e2e_tests/migration.serverRootPath.config.ts",
     "knip": "knip",
     "knip:fix": "knip --fix",
     "gen:api": "node scripts/gen-api-types.mjs"