diff --git a/.circleci/config.yml b/.circleci/config.yml index a8a33335ad..dbeb412506 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2690,6 +2690,122 @@ jobs: path: ui/litellm-dashboard/playwright-report destination: e2e-playwright-report + e2e_ui_testing_server_root_path: + docker: + - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: e2euser + POSTGRES_PASSWORD: e2epassword + POSTGRES_DB: litellm_e2e + resource_class: large + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" + CI: "true" + # The whole job exercises the proxy mounted under a prefix. SERVER_ROOT_PATH + # is read both by the proxy at boot (to rewrite the built UI bundle in place) + # and by migration.serverRootPath.config.ts, which refuses to run without it. + SERVER_ROOT_PATH: "/litellm" + steps: + - checkout + - setup_google_dns + - install_uv + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - run: + name: Install Python dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma + - save_cache: + key: v1-uv-cache-{{ checksum "uv.lock" }} + paths: + - ~/.cache/uv + - restore_cache: + keys: + - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - run: + name: Install Node dependencies and Playwright + command: | + cd ui/litellm-dashboard + npm ci + npx playwright install chromium + - save_cache: + key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/node_modules + - ~/.cache/ms-playwright + - run: + name: Build UI from source + command: | + cd ui/litellm-dashboard + npm run build + rm -rf ../../litellm/proxy/_experimental/out + mv out ../../litellm/proxy/_experimental/out + find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do + d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" + done + - wait_for_service: + url: tcp://localhost:5432 + timeout: "30" + - run: + name: Push Prisma schema + command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + - run: + name: Seed database + command: | + PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ + -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql + - run: + name: Start mock LLM server + command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py + background: true + - run: + name: Start LiteLLM proxy under a server root path + environment: + LITELLM_MASTER_KEY: "sk-1234" + MOCK_LLM_URL: "http://127.0.0.1:8090/v1" + DISABLE_SCHEMA_UPDATE: "true" + # Output flows to this step's own log, so a boot crash is visible here + # rather than swallowed by a downstream readiness probe. + command: | + LITELLM_LICENSE="$LITELLM_LICENSE" \ + uv run --no-sync python -m litellm.proxy.proxy_cli \ + --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \ + --port 4000 + background: true + - run: + name: Wait for prefixed proxy to be ready + command: | + for i in $(seq 1 60); do + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer sk-1234" http://127.0.0.1:4000/litellm/health 2>/dev/null || true) + if [ "$HTTP_CODE" = "200" ]; then + echo "Prefixed proxy is ready" + exit 0 + fi + sleep 2 + done + echo "Prefixed proxy failed to start; see the 'Start LiteLLM proxy under a server root path' step for the boot log" + exit 1 + - run: + name: Run migration smoke under SERVER_ROOT_PATH + command: | + cd ui/litellm-dashboard + LITELLM_LICENSE="$LITELLM_LICENSE" \ + npx playwright test --config e2e_tests/migration.serverRootPath.config.ts + no_output_timeout: 10m + - store_artifacts: + path: ui/litellm-dashboard/test-results + destination: e2e-server-root-path-test-results + - store_artifacts: + path: ui/litellm-dashboard/playwright-report + destination: e2e-server-root-path-playwright-report + build_docker_database_image: machine: image: ubuntu-2204:2024.04.1 @@ -2795,6 +2911,8 @@ workflows: filters: *main_branches - e2e_ui_testing: filters: *main_branches + - e2e_ui_testing_server_root_path: + filters: *main_branches - build_and_test: requires: - build_docker_database_image diff --git a/ui/litellm-dashboard/e2e_tests/fixtures/migratedPages.ts b/ui/litellm-dashboard/e2e_tests/fixtures/migratedPages.ts new file mode 100644 index 0000000000..f2ba66147e --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/fixtures/migratedPages.ts @@ -0,0 +1,16 @@ +/** + * Source of truth for the App Router migration smoke (tests/migration/migratedPages.spec.ts). + * + * Add a route segment here once its migration has MERGED to the branch under test. + * Both suites pick it up automatically: + * - default mount: npm run e2e:migration + * - server-root-path mount: SERVER_ROOT_PATH=/ npm run e2e:migration:root + * + * Keep this in lockstep with MIGRATED_PAGES in src/utils/migratedPages.ts. + * Pending (uncomment as each PR lands): playground, and the leaf-pages batch + * (budgets, caching, cost-tracking, guardrails, guardrails-monitor, logs, + * mcp-servers, memory, policies, projects, prompts, search-tools, skills, + * tag-management, tool-policies, transform-request, ui-theme, vector-stores, + * workflows, access-groups). + */ +export const MIGRATED_E2E_SEGMENTS: string[] = ["api-reference"]; diff --git a/ui/litellm-dashboard/e2e_tests/globalSetup.ts b/ui/litellm-dashboard/e2e_tests/globalSetup.ts index 8f80f57bd7..0b3fa7e880 100644 --- a/ui/litellm-dashboard/e2e_tests/globalSetup.ts +++ b/ui/litellm-dashboard/e2e_tests/globalSetup.ts @@ -4,17 +4,18 @@ import * as fs from "fs"; async function globalSetup() { const browser = await chromium.launch(); + const rootPath = process.env.SERVER_ROOT_PATH ?? ""; for (const role of Object.values(Role)) { const { email, password } = users[role]; const storagePath = STORAGE_PATHS[role]; const page = await browser.newPage(); try { - await page.goto("http://localhost:4000/ui/login"); + await page.goto(`http://localhost:4000${rootPath}/ui/login`); await page.getByPlaceholder("Enter your username").fill(email); await page.getByPlaceholder("Enter your password").fill(password); await page.getByRole("button", { name: "Login", exact: true }).click(); - await page.waitForURL((url) => url.pathname.startsWith("/ui") && !url.pathname.includes("/login"), { + await page.waitForURL((url) => url.pathname.startsWith(`${rootPath}/ui`) && !url.pathname.includes("/login"), { timeout: 30_000, }); await expect(page.locator("a", { hasText: "Virtual Keys" })).toBeVisible({ timeout: 30_000 }); diff --git a/ui/litellm-dashboard/e2e_tests/migration.serverRootPath.config.ts b/ui/litellm-dashboard/e2e_tests/migration.serverRootPath.config.ts new file mode 100644 index 0000000000..205348463c --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/migration.serverRootPath.config.ts @@ -0,0 +1,38 @@ +import { defineConfig, devices } from "@playwright/test"; + +/** + * App Router migration smoke under a non-root mount. Boot the proxy with the same + * SERVER_ROOT_PATH (e.g. SERVER_ROOT_PATH=/litellm) and a UI built for it before + * running. globalSetup logs in at `${SERVER_ROOT_PATH}/ui/login` so the admin + * storage state is valid under the prefix. + */ +if (!process.env.SERVER_ROOT_PATH) { + throw new Error( + "migration.serverRootPath.config.ts requires SERVER_ROOT_PATH to be set (e.g. SERVER_ROOT_PATH=/litellm). " + + "Without it this config silently re-runs the default mount and never exercises the prefix. " + + "For the root-less run use the default playwright.config.ts (npm run e2e:migration).", + ); +} + +export default defineConfig({ + testDir: "./tests/migration", + testMatch: ["migratedPages.spec.ts"], + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.CI ? 1 : undefined, + reporter: "list", + use: { + baseURL: "http://localhost:4000", + trace: "on-first-retry", + actionTimeout: 15 * 1000, + navigationTimeout: 30 * 1000, + launchOptions: { + slowMo: process.env.SLOWMO ? parseInt(process.env.SLOWMO, 10) || 0 : 0, + }, + }, + projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }], + timeout: 3 * 60 * 1000, + expect: { timeout: 10 * 1000 }, + globalSetup: require.resolve("./globalSetup"), +}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/migration/README.md b/ui/litellm-dashboard/e2e_tests/tests/migration/README.md new file mode 100644 index 0000000000..4b3a391d42 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/migration/README.md @@ -0,0 +1,33 @@ +# App Router migration smoke + +A growing E2E smoke for pages migrated from the legacy `?page=` switch to App +Router path routes. For each migrated page it clicks the page's sidebar link, checks +the URL is the path route and the page renders, reloads it, then clicks off to a +legacy page and back to confirm navigation still works. It runs in two situations: +the default mount and a non-root `SERVER_ROOT_PATH` mount. + +## Adding a page + +When a page's migration merges, add its route segment to +`e2e_tests/fixtures/migratedPages.ts` (keep it in lockstep with `MIGRATED_PAGES` +in `src/utils/migratedPages.ts`). Both suites pick it up automatically. + +## Running + +Build the UI into the proxy and start the proxy first (the suite runs against +`http://localhost:4000`). + +Default mount: + +``` +npm run e2e:migration +``` + +Non-root mount (build and boot the proxy with the same root path, e.g. `/litellm`): + +``` +SERVER_ROOT_PATH=/litellm npm run e2e:migration:root +``` + +`globalSetup` logs in once per role; the admin storage state is reused for these +tests. Under a non-root mount it logs in at `${SERVER_ROOT_PATH}/ui/login`. diff --git a/ui/litellm-dashboard/e2e_tests/tests/migration/migratedPages.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/migration/migratedPages.spec.ts new file mode 100644 index 0000000000..98f4fee145 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/migration/migratedPages.spec.ts @@ -0,0 +1,101 @@ +import { test, expect, type Page } from "@playwright/test"; +import { MIGRATED_E2E_SEGMENTS } from "../../fixtures/migratedPages"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { dismissFeedbackPopup } from "../../helpers/navigation"; + +/** + * App Router migration smoke as a user journey: start where the proxy lands you, + * click a migrated page in the sidebar, confirm it routed and rendered, reload it + * (the check a wrong server_root_path breaks), bounce to a legacy page and back, + * and, once two pages are migrated, navigate directly between two migrated pages. + * + * Driven by MIGRATED_E2E_SEGMENTS, so it grows as pages are migrated. Set + * SERVER_ROOT_PATH (e.g. "/litellm") to exercise the non-root mount; leave it + * unset for the default mount. Boot the proxy with the matching value first. + */ +const ROOT = process.env.SERVER_ROOT_PATH ?? ""; + +const esc = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +const pathRe = (segment: string) => new RegExp(`${esc(ROOT)}/ui/${esc(segment)}/?($|\\?)`); +const legacyAnchor = (page: Page) => page.locator("a", { hasText: "Virtual Keys" }); + +/** The dashboard shell is present (sidebar rendered); page didn't 404 / crash. */ +async function expectRendered(page: Page) { + await expect(legacyAnchor(page)).toBeVisible({ timeout: 20_000 }); +} + +/** + * Click a migrated page's sidebar link. Migrated items render as ; + * nested ones live under collapsible submenus, so expand submenus until the link is clickable. + */ +async function clickSidebar(page: Page, segment: string) { + const link = page.locator(`a[href$="/ui/${segment}"]`).first(); + for (let i = 0; i < 8 && !(await link.isVisible().catch(() => false)); i++) { + const collapsedSubmenu = page + .locator(".ant-menu-submenu:not(.ant-menu-submenu-open) > .ant-menu-submenu-title") + .first(); + if (!(await collapsedSubmenu.isVisible().catch(() => false))) break; + await collapsedSubmenu.click(); + await page.waitForTimeout(250); + } + await link.click(); +} + +test.use({ storageState: ADMIN_STORAGE_PATH }); + +test.describe("App Router migrated pages", () => { + for (const segment of MIGRATED_E2E_SEGMENTS) { + test(`${segment}: sidebar nav, reload, and round-trip with a legacy page`, async ({ page }) => { + const pageErrors: string[] = []; + page.on("pageerror", (e) => pageErrors.push(String(e))); + + // 1. Start where the proxy lands us. + await page.goto(`${ROOT}/ui/`); + await dismissFeedbackPopup(page); + await expectRendered(page); + + // 2. Click the migrated page in the sidebar -> path route + rendered. + await clickSidebar(page, segment); + await expect(page).toHaveURL(pathRe(segment)); + await expectRendered(page); + // 3. Reload the path route directly; a wrong server_root_path 404s here. + await page.reload(); + await dismissFeedbackPopup(page); + await expect(page).toHaveURL(pathRe(segment)); + await expectRendered(page); + // 4. Click off to a legacy (not-yet-migrated) page. + await legacyAnchor(page).click(); + await expect(page).toHaveURL(new RegExp(`${esc(ROOT)}/ui/\\?page=api-keys`)); + await dismissFeedbackPopup(page); + await expectRendered(page); + // 5. Click back to the migrated page. + await clickSidebar(page, segment); + await expect(page).toHaveURL(pathRe(segment)); + await expectRendered(page); + expect(pageErrors, `page errors during ${segment} journey`).toEqual([]); + }); + } + + test("navigates directly between two migrated pages", async ({ page }) => { + test.skip(MIGRATED_E2E_SEGMENTS.length < 2, "needs >= 2 migrated pages"); + const [first, second] = MIGRATED_E2E_SEGMENTS; + const pageErrors: string[] = []; + page.on("pageerror", (e) => pageErrors.push(String(e))); + + await page.goto(`${ROOT}/ui/`); + await dismissFeedbackPopup(page); + + await clickSidebar(page, first); + await expect(page).toHaveURL(pathRe(first)); + await expectRendered(page); + await clickSidebar(page, second); + await expect(page).toHaveURL(pathRe(second)); + await expectRendered(page); + // Back to the first migrated page. + await clickSidebar(page, first); + await expect(page).toHaveURL(pathRe(first)); + await expectRendered(page); + + expect(pageErrors, "page errors during migrated -> migrated nav").toEqual([]); + }); +}); diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 623389e76e..c9795cf7c6 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -16,6 +16,8 @@ "format:check": "prettier --check .", "e2e": "playwright test --config e2e_tests/playwright.config.ts", "e2e:ui": "playwright test --ui --config e2e_tests/playwright.config.ts", + "e2e:migration": "playwright test e2e_tests/tests/migration/migratedPages.spec.ts --config e2e_tests/playwright.config.ts", + "e2e:migration:root": "playwright test --config e2e_tests/migration.serverRootPath.config.ts", "knip": "knip", "knip:fix": "knip --fix", "gen:api": "node scripts/gen-api-types.mjs"