From d09d98a70a8bd5f2b9d04998bb6a31192be7965f Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 8 Apr 2026 11:51:15 -0700 Subject: [PATCH 1/5] [Feature] E2E UI tests: proxy-admin team and key management with CI integration Add Playwright E2E tests covering proxy admin team and key management workflows, with a self-contained test runner and CircleCI integration. Tests cover: create team, invite user, edit/delete team members, create key in team, regenerate key, update TPM/RPM limits, delete key, and verify internal user keys are visible. Infrastructure: run_e2e.sh builds the UI from source before starting the proxy, ensuring tests always run against the latest UI changes. Added data-testid attributes to key UI components for reliable selectors. --- .circleci/config.yml | 139 +++++++++++--- .../e2e_tests/fixtures/config.yml | 16 ++ .../fixtures/mock_llm_server/server.py | 120 ++++++++++++ ui/litellm-dashboard/e2e_tests/globalSetup.ts | 45 +++-- .../e2e_tests/helpers/navigation.ts | 21 +- ui/litellm-dashboard/e2e_tests/run_e2e.sh | 180 ++++++++++++++++++ .../e2e_tests/tests/proxy-admin/keys.spec.ts | 124 ++++++++++++ .../e2e_tests/tests/proxy-admin/teams.spec.ts | 136 +++++++++++++ .../components/TeamsTable/TeamsTable.tsx | 1 + .../components/modals/CreateTeamModal.tsx | 6 +- .../src/components/OldTeams.tsx | 8 +- .../common_components/team_dropdown.tsx | 1 + .../common_components/user_search_modal.tsx | 1 + .../organisms/create_key_button.tsx | 2 +- 14 files changed, 752 insertions(+), 48 deletions(-) create mode 100644 ui/litellm-dashboard/e2e_tests/fixtures/config.yml create mode 100644 ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py create mode 100755 ui/litellm-dashboard/e2e_tests/run_e2e.sh create mode 100644 ui/litellm-dashboard/e2e_tests/tests/proxy-admin/keys.spec.ts create mode 100644 ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index 85c5788693..a9d395e768 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3074,6 +3074,113 @@ jobs: CI=true npm run test -- --run \ --pool forks --poolOptions.forks.maxForks=8 + ui_e2e_tests: + docker: + - image: cimg/python:3.12-browsers + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + - image: cimg/postgres:16.0 + environment: + POSTGRES_USER: e2euser + POSTGRES_PASSWORD: e2epassword + POSTGRES_DB: litellm_e2e + resource_class: large + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" + CI: "true" + steps: + - checkout + - setup_google_dns + - restore_cache: + keys: + - ui-e2e-py-deps-v1-{{ checksum "requirements.txt" }} + - run: + name: Install Python dependencies + command: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "prisma==0.11.0" + prisma generate --schema litellm/proxy/schema.prisma + - save_cache: + key: ui-e2e-py-deps-v1-{{ checksum "requirements.txt" }} + paths: + - ~/.local/lib + - ~/.local/bin + - restore_cache: + keys: + - ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - run: + name: Install Node dependencies and Playwright + command: | + cd ui/litellm-dashboard + npm ci + npx playwright install chromium --with-deps + - save_cache: + key: ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/node_modules + - run: + name: Build UI from source + command: | + cd ui/litellm-dashboard + npm run build + cp -r out/ ../../litellm/proxy/_experimental/out/ + - run: + name: Wait for PostgreSQL + command: dockerize -wait tcp://localhost:5432 -timeout 30s + - run: + name: Push Prisma schema + command: prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + - run: + name: Seed database + command: | + PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ + -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql + - run: + name: Start mock LLM server + command: python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py + background: true + - run: + name: Start LiteLLM proxy + environment: + LITELLM_MASTER_KEY: "sk-1234" + MOCK_LLM_URL: "http://127.0.0.1:8090/v1" + DISABLE_SCHEMA_UPDATE: "true" + SERVER_ROOT_PATH: "" + PROXY_LOGOUT_URL: "" + command: | + python -m litellm.proxy.proxy_cli \ + --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \ + --port 4000 + background: true + - run: + name: Wait for proxy to be ready + command: | + for i in $(seq 1 60); do + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true) + if [ "$HTTP_CODE" = "200" ]; then + echo "Proxy is ready" + exit 0 + fi + sleep 2 + done + echo "Proxy failed to start" + exit 1 + - run: + name: Run Playwright E2E tests + command: | + cd ui/litellm-dashboard + npx playwright test --config e2e_tests/playwright.config.ts + no_output_timeout: 10m + - store_artifacts: + path: ui/litellm-dashboard/test-results + destination: e2e-test-results + - store_artifacts: + path: ui/litellm-dashboard/playwright-report + destination: e2e-playwright-report + build_docker_database_image: machine: image: ubuntu-2204:2024.04.1 @@ -3401,32 +3508,12 @@ workflows: only: - main - /litellm_.*/ - # - e2e_ui_testing: - # name: e2e_ui_testing_chromium - # browser: chromium - # context: e2e_ui_tests - # requires: - # - ui_build - # - build_docker_database_image - # - prisma_schema_sync - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - e2e_ui_testing: - # name: e2e_ui_testing_firefox - # browser: firefox - # context: e2e_ui_tests - # requires: - # - ui_build - # - build_docker_database_image - # - prisma_schema_sync - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ + - ui_e2e_tests: + filters: + branches: + only: + - main + - /litellm_.*/ - build_and_test: requires: - build_docker_database_image diff --git a/ui/litellm-dashboard/e2e_tests/fixtures/config.yml b/ui/litellm-dashboard/e2e_tests/fixtures/config.yml new file mode 100644 index 0000000000..438c236b03 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/fixtures/config.yml @@ -0,0 +1,16 @@ +model_list: + - model_name: fake-openai-gpt-4 + litellm_params: + model: openai/fake-gpt-4 + api_base: os.environ/MOCK_LLM_URL + api_key: fake-key + - model_name: fake-anthropic-claude + litellm_params: + model: openai/fake-claude + api_base: os.environ/MOCK_LLM_URL + api_key: fake-key + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL + store_prompts_in_spend_logs: true diff --git a/ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py b/ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py new file mode 100644 index 0000000000..8e92065c69 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py @@ -0,0 +1,120 @@ +""" +Mock LLM server for UI e2e tests. +Responds to OpenAI-format endpoints with canned responses. +""" + +import time +import json +import uuid + +import uvicorn +from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import StreamingResponse + + +app = FastAPI(title="Mock LLM Server") +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/health") +async def health(): + return {"status": "ok"} + + +@app.get("/v1/models") +@app.get("/models") +async def list_models(): + return { + "object": "list", + "data": [ + {"id": "fake-gpt-4", "object": "model", "owned_by": "mock"}, + {"id": "fake-claude", "object": "model", "owned_by": "mock"}, + ], + } + + +@app.post("/v1/chat/completions") +@app.post("/chat/completions") +async def chat_completions(request: Request): + body = await request.json() + model = body.get("model", "mock-model") + stream = body.get("stream", False) + + response_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" + created = int(time.time()) + + if stream: + + async def stream_generator(): + chunk = { + "id": response_id, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "content": "This is a mock response.", + }, + "finish_reason": None, + } + ], + } + yield f"data: {json.dumps(chunk)}\n\n" + + done_chunk = { + "id": response_id, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + } + yield f"data: {json.dumps(done_chunk)}\n\n" + yield "data: [DONE]\n\n" + + return StreamingResponse(stream_generator(), media_type="text/event-stream") + + return { + "id": response_id, + "object": "chat.completion", + "created": created, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "This is a mock response."}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18}, + } + + +@app.post("/v1/embeddings") +@app.post("/embeddings") +async def embeddings(request: Request): + body = await request.json() + inputs = body.get("input", [""]) + if isinstance(inputs, str): + inputs = [inputs] + return { + "object": "list", + "data": [ + {"object": "embedding", "index": i, "embedding": [0.0] * 1536} + for i in range(len(inputs)) + ], + "model": body.get("model", "mock-embedding"), + "usage": {"prompt_tokens": 5, "total_tokens": 5}, + } + + +if __name__ == "__main__": + uvicorn.run(app, host="127.0.0.1", port=8090) diff --git a/ui/litellm-dashboard/e2e_tests/globalSetup.ts b/ui/litellm-dashboard/e2e_tests/globalSetup.ts index 44d50a49af..6ff5522244 100644 --- a/ui/litellm-dashboard/e2e_tests/globalSetup.ts +++ b/ui/litellm-dashboard/e2e_tests/globalSetup.ts @@ -1,17 +1,40 @@ -import { chromium } from "@playwright/test"; -import { users } from "./fixtures/users"; -import { Role } from "./fixtures/roles"; +import { chromium, expect } from "@playwright/test"; +import { users, Role, STORAGE_PATHS } from "./fixtures/users"; +import * as fs from "fs"; async function globalSetup() { const browser = await chromium.launch(); - const page = await browser.newPage(); - await page.goto("http://localhost:4000/ui/login"); - await page.getByPlaceholder("Enter your username").fill(users[Role.ProxyAdmin].email); - await page.getByPlaceholder("Enter your password").fill(users[Role.ProxyAdmin].password); - const loginButton = page.getByRole("button", { name: "Login", exact: true }); - await loginButton.click(); - await page.waitForSelector("text=Virtual Keys"); - await page.context().storageState({ path: "admin.storageState.json" }); + + for (const role of Object.values(Role)) { + const { email, password } = users[role]; + const storagePath = STORAGE_PATHS[role]; + const page = await browser.newPage(); + try { + await page.goto("http://localhost:4000/ui/login"); + await page.getByPlaceholder("Enter your username").fill(email); + await page.getByPlaceholder("Enter your password").fill(password); + await page.getByRole("button", { name: "Login", exact: true }).click(); + await page.waitForURL( + (url) => url.pathname.startsWith("/ui") && !url.pathname.includes("/login"), + { timeout: 30_000 }, + ); + await expect(page.locator("a", { hasText: "Virtual Keys" })).toBeVisible({ timeout: 30_000 }); + // Dismiss feedback popup if present + const dismiss = page.getByText("Don't ask me again"); + if (await dismiss.isVisible({ timeout: 1_500 }).catch(() => false)) { + await dismiss.click(); + } + await page.context().storageState({ path: storagePath }); + } catch (e) { + fs.mkdirSync("test-results", { recursive: true }); + await page.screenshot({ path: `test-results/global-setup-${role}-failure.png`, fullPage: true }); + console.error(`Global setup failed for role ${role}. Screenshot saved. URL: ${page.url()}`); + throw e; + } finally { + await page.close(); + } + } + await browser.close(); } diff --git a/ui/litellm-dashboard/e2e_tests/helpers/navigation.ts b/ui/litellm-dashboard/e2e_tests/helpers/navigation.ts index 919e516b35..3eb0dc9b24 100644 --- a/ui/litellm-dashboard/e2e_tests/helpers/navigation.ts +++ b/ui/litellm-dashboard/e2e_tests/helpers/navigation.ts @@ -1,12 +1,25 @@ import { Page } from "../fixtures/pages"; -import { Page as PlaywrightPage } from "@playwright/test"; +import { Page as PlaywrightPage, expect } from "@playwright/test"; /** * Navigates to a specific page using the page query parameter. - * Uses relative path which will be resolved against the baseURL configured in playwright.config.ts - * @param page - The Playwright page object - * @param pageEnum - The page enum value to navigate to + * Waits for the sidebar to be visible before returning. */ export async function navigateToPage(page: PlaywrightPage, pageEnum: Page): Promise { await page.goto(`/ui?page=${pageEnum}`); + await page.waitForLoadState("networkidle"); + // Dismiss the "Quick feedback" popup if it appears + await dismissFeedbackPopup(page); +} + +/** + * Dismiss the "Quick feedback" popup that may appear on any page. + */ +export async function dismissFeedbackPopup(page: PlaywrightPage): Promise { + const dismissButton = page.getByText("Don't ask me again"); + if (await dismissButton.isVisible({ timeout: 1_500 }).catch(() => false)) { + await dismissButton.click(); + // Wait for the popup to disappear + await expect(dismissButton).not.toBeVisible({ timeout: 2_000 }).catch(() => {}); + } } diff --git a/ui/litellm-dashboard/e2e_tests/run_e2e.sh b/ui/litellm-dashboard/e2e_tests/run_e2e.sh new file mode 100755 index 0000000000..9e97914658 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/run_e2e.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ================================================================ +# UI E2E Test Runner (Consolidated) +# Starts postgres, seeds DB, starts mock + proxy, runs Playwright. +# All tests target the proxy on port 4000 (which serves both API +# and UI from the built Next.js static export). +# +# Usage: +# ./run_e2e.sh # Run once +# ./run_e2e.sh --repeat-each=5 # Run each test 5 times +# ./run_e2e.sh --headed # Run with browser visible +# +# In CI (CI=true), expects: +# - PostgreSQL already running on 127.0.0.1:5432 +# - DATABASE_URL already set +# - Python/Poetry already installed +# - Node.js/npx already available +# ================================================================ + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DASHBOARD_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +IS_CI="${CI:-false}" +CONTAINER_NAME="litellm-e2e-postgres-$$" +MOCK_PID="" +PROXY_PID="" + +# --- Ensure common tool paths are available (local dev only) --- +if [ "$IS_CI" = "false" ]; then + for p in /usr/local/bin /opt/homebrew/bin "$HOME/.local/bin" /opt/homebrew/opt/postgresql@14/bin /opt/homebrew/opt/libpq/bin; do + [ -d "$p" ] && export PATH="$p:$PATH" + done + [ -s "$HOME/.nvm/nvm.sh" ] && source "$HOME/.nvm/nvm.sh" +fi + +# --- Cleanup on exit --- +cleanup() { + echo "Cleaning up..." + [ -n "$MOCK_PID" ] && kill "$MOCK_PID" 2>/dev/null || true + [ -n "$PROXY_PID" ] && kill "$PROXY_PID" 2>/dev/null || true + if [ "$IS_CI" = "false" ]; then + docker stop "$CONTAINER_NAME" 2>/dev/null || true + fi + echo "Done." +} +trap cleanup EXIT INT TERM + +# --- Pre-flight checks --- +for cmd in python3 npx poetry; do + command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; } +done + +# --- Database setup --- +if [ "$IS_CI" = "false" ]; then + for cmd in docker psql; do + command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; } + done + for port in 4000 5432 8090; do + if lsof -ti ":$port" >/dev/null 2>&1; then + echo "Error: port $port is in use" + exit 1 + fi + done + + export POSTGRES_USER="e2euser" + export POSTGRES_PASSWORD="$(openssl rand -hex 32)" + export POSTGRES_DB="litellm_e2e" + export DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:5432/${POSTGRES_DB}" + + echo "=== Starting PostgreSQL ===" + docker run -d --rm --name "$CONTAINER_NAME" \ + -e POSTGRES_USER -e POSTGRES_PASSWORD -e POSTGRES_DB \ + -p 127.0.0.1:5432:5432 \ + postgres:16 + + echo "Waiting for PostgreSQL..." + for i in $(seq 1 30); do + if PGPASSWORD="$POSTGRES_PASSWORD" pg_isready -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then + break + fi + sleep 1 + done +else + echo "=== Using CI PostgreSQL service ===" + : "${DATABASE_URL:?DATABASE_URL must be set in CI}" +fi + +# --- Credentials --- +export LITELLM_MASTER_KEY="sk-1234" +export MOCK_LLM_URL="http://127.0.0.1:8090/v1" +export DISABLE_SCHEMA_UPDATE="true" +# Ensure the proxy serves UI at /ui (not behind a subpath) +export SERVER_ROOT_PATH="" +# Prevent logout from redirecting to an external URL +export PROXY_LOGOUT_URL="" + +# --- Rebuild UI from source --- +echo "=== Building UI from source ===" +cd "$DASHBOARD_DIR" +npm install --silent 2>/dev/null || true +npm run build +# Copy the fresh build to the proxy's static UI directory +cp -r "$DASHBOARD_DIR/out/" "$REPO_ROOT/litellm/proxy/_experimental/out/" +echo "UI build copied to proxy static directory" + +# --- Python environment --- +echo "=== Setting up Python environment ===" +cd "$REPO_ROOT" +if ! poetry run python3 -c "import prisma" 2>/dev/null; then + echo "Installing Python dependencies (first run)..." + poetry install --with dev,proxy-dev --extras "proxy" --quiet + poetry run pip install nodejs-wheel-binaries 2>/dev/null || true + poetry run prisma generate --schema litellm/proxy/schema.prisma +fi + +echo "=== Pushing Prisma schema to database ===" +poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + +# --- Mock LLM server --- +echo "=== Starting mock LLM server ===" +poetry run python3 "$SCRIPT_DIR/fixtures/mock_llm_server/server.py" & +MOCK_PID=$! + +for i in $(seq 1 15); do + if curl -sf http://127.0.0.1:8090/health >/dev/null 2>&1; then break; fi + sleep 1 +done + +# --- LiteLLM proxy --- +echo "=== Starting LiteLLM proxy ===" +cd "$REPO_ROOT" +poetry run python3 -m litellm.proxy.proxy_cli \ + --config "$SCRIPT_DIR/fixtures/config.yml" \ + --port 4000 & +PROXY_PID=$! + +echo "Waiting for proxy..." +PROXY_READY=0 +for i in $(seq 1 180); do + if ! kill -0 "$PROXY_PID" 2>/dev/null; then + echo "Error: proxy process exited unexpectedly" + exit 1 + fi + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true) + if [ "$HTTP_CODE" = "200" ]; then + PROXY_READY=1 + break + fi + sleep 1 +done +if [ "$PROXY_READY" -ne 1 ]; then + echo "Error: proxy did not become healthy within 180 seconds" + exit 1 +fi +echo "Proxy is ready." + +# --- Seed database --- +echo "=== Seeding database ===" +DB_USER=$(echo "$DATABASE_URL" | sed -n 's|.*://\([^:]*\):.*|\1|p') +DB_PASS=$(echo "$DATABASE_URL" | sed -n 's|.*://[^:]*:\([^@]*\)@.*|\1|p') +DB_HOST=$(echo "$DATABASE_URL" | sed -n 's|.*@\([^:]*\):.*|\1|p') +DB_PORT=$(echo "$DATABASE_URL" | sed -n 's|.*:\([0-9]*\)/.*|\1|p') +DB_NAME=$(echo "$DATABASE_URL" | sed -n 's|.*/\([^?]*\).*|\1|p') + +PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \ + -f "$SCRIPT_DIR/fixtures/seed.sql" + +# --- Playwright --- +echo "=== Installing Playwright dependencies ===" +cd "$DASHBOARD_DIR" +npm install --silent 2>/dev/null || true +npx playwright install chromium --with-deps 2>/dev/null || npx playwright install chromium + +echo "=== Running Playwright tests ===" +npx playwright test --config e2e_tests/playwright.config.ts "$@" +EXIT_CODE=$? + +exit $EXIT_CODE diff --git a/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/keys.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/keys.spec.ts new file mode 100644 index 0000000000..aba37e25be --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/keys.spec.ts @@ -0,0 +1,124 @@ +import { test, expect } from "@playwright/test"; +import { + ADMIN_STORAGE_PATH, + E2E_DELETE_KEY_ALIAS, + E2E_REGENERATE_KEY_ALIAS, + E2E_UPDATE_LIMITS_KEY_ALIAS, + E2E_INTERNAL_USER_KEY_ALIAS, + E2E_TEAM_CRUD_ALIAS, +} from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; + +test.describe("Proxy Admin - Keys", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Create a key in a team", async ({ page }) => { + await navigateToPage(page, Page.ApiKeys); + await dismissFeedbackPopup(page); + + // Click "+ Create New Key" button + await page.getByRole("button", { name: /Create New Key/i }).click(); + + // Wait for the key creation modal + await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 }); + + // Fill key name (has data-testid="base-input" in the built UI) + const keyName = `e2e-admin-key-${Date.now()}`; + await page.getByTestId("base-input").fill(keyName); + + // Select team — the team dropdown has placeholder "Search or select a team" + const teamSelect = page.locator(".ant-select", { hasText: "Search or select a team" }); + await teamSelect.click(); + await page.keyboard.type(E2E_TEAM_CRUD_ALIAS); + await page.locator(".ant-select-dropdown:visible").getByText(E2E_TEAM_CRUD_ALIAS).first().click(); + + // Select models + await page.locator(".ant-select-selection-overflow").click(); + await page.locator(".ant-select-dropdown:visible").getByText("All Team Models").click(); + await page.keyboard.press("Escape"); + + // Submit + await page.getByRole("button", { name: "Create Key", exact: true }).click(); + + // Success shows "Save your Key" in a second dialog + await expect(page.getByText("Save your Key")).toBeVisible({ timeout: 10_000 }); + await page.keyboard.press("Escape"); + + // Verify the new key appears in the table + await expect(page.getByText(keyName)).toBeVisible({ timeout: 10_000 }); + }); + + test("Regenerate key", async ({ page }) => { + await navigateToPage(page, Page.ApiKeys); + await dismissFeedbackPopup(page); + + // Key IDs are rendered as buttons in the table + const keyRow = page.locator("tr", { hasText: E2E_REGENERATE_KEY_ALIAS }); + await expect(keyRow).toBeVisible({ timeout: 10_000 }); + await keyRow.locator("button").first().click(); + + await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 }); + + await page.getByRole("button", { name: "Regenerate Key" }).click(); + await page.getByRole("button", { name: "Regenerate", exact: true }).click(); + + // Success shows "Copy Virtual Key" button in the regenerated key dialog + await expect(page.getByText("Copy Virtual Key")).toBeVisible({ timeout: 10_000 }); + }); + + test("Update key TPM and RPM limits", async ({ page }) => { + await navigateToPage(page, Page.ApiKeys); + await dismissFeedbackPopup(page); + + const keyRow = page.locator("tr", { hasText: E2E_UPDATE_LIMITS_KEY_ALIAS }); + await expect(keyRow).toBeVisible({ timeout: 10_000 }); + await keyRow.locator("button").first().click(); + + await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 }); + + await page.getByRole("tab", { name: "Settings" }).click(); + await page.getByRole("button", { name: "Edit Settings" }).click(); + + await page.getByRole("spinbutton", { name: "TPM Limit" }).fill("123"); + await page.getByRole("spinbutton", { name: "RPM Limit" }).fill("456"); + await page.getByRole("button", { name: "Save Changes" }).click(); + + await expect( + page.getByRole("paragraph").filter({ hasText: "TPM: 123" }) + ).toBeVisible({ timeout: 10_000 }); + await expect( + page.getByRole("paragraph").filter({ hasText: "RPM: 456" }) + ).toBeVisible({ timeout: 10_000 }); + }); + + test("Delete key", async ({ page }) => { + await navigateToPage(page, Page.ApiKeys); + await dismissFeedbackPopup(page); + + const keyRow = page.locator("tr", { hasText: E2E_DELETE_KEY_ALIAS }); + await expect(keyRow).toBeVisible({ timeout: 10_000 }); + await keyRow.locator("button").first().click(); + + await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 }); + + await page.getByRole("button", { name: "Delete Key" }).click(); + + const modal = page.locator(".ant-modal:visible"); + await expect(modal).toBeVisible({ timeout: 5_000 }); + await modal.locator("input").fill(E2E_DELETE_KEY_ALIAS); + + const deleteButton = modal.getByRole("button", { name: "Delete", exact: true }); + await expect(deleteButton).toBeEnabled(); + await deleteButton.click(); + + await expect(page.getByText(/Key deleted/i).first()).toBeVisible({ timeout: 10_000 }); + }); + + test("See internal user keys in team", async ({ page }) => { + await navigateToPage(page, Page.ApiKeys); + await dismissFeedbackPopup(page); + + await expect(page.getByText(E2E_INTERNAL_USER_KEY_ALIAS)).toBeVisible({ timeout: 10_000 }); + }); +}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts new file mode 100644 index 0000000000..19c33a2cd7 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts @@ -0,0 +1,136 @@ +import { test, expect } from "@playwright/test"; +import { + ADMIN_STORAGE_PATH, + E2E_TEAM_CRUD_ID, + E2E_TEAM_DELETE_ALIAS, + E2E_TEAM_NO_ADMIN_ID, + E2E_TEAM_ORG_ID, +} from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; + +/** + * Click on a team ID in the table. Team IDs are rendered differently depending + * on the component version — try button first (Tremor Button), fall back to + * clickable span (OldTeams Typography.Text). + */ +async function clickTeamId(page: import("@playwright/test").Page, teamId: string) { + const idPrefix = teamId.slice(0, 7); + // The team ID is either a Button or a clickable span containing the first 7 chars + const cell = page.locator("td").filter({ hasText: teamId }).first(); + await expect(cell).toBeVisible({ timeout: 10_000 }); + await cell.click(); + await expect(page.getByText("Back to Teams")).toBeVisible({ timeout: 10_000 }); +} + +test.describe("Proxy Admin - Teams", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Create a team", async ({ page }) => { + await navigateToPage(page, Page.Teams); + await dismissFeedbackPopup(page); + + const uniqueAlias = `e2e-created-team-${Date.now()}`; + + // Click the Create Team button — accessible name includes "Create Team" + await page.getByRole("button", { name: /Create Team/i }).first().click(); + + // Wait for the Create Team modal + const dialog = page.locator(".ant-modal:visible"); + await expect(dialog).toBeVisible({ timeout: 5_000 }); + + // Fill Team Name — the input has id="team_alias" + await dialog.locator("#team_alias").fill(uniqueAlias); + + // Select models — the models multi-select is inside the modal + // Click to open dropdown, select "All Proxy Models" + await dialog.locator(".ant-select-selection-overflow").first().click(); + await page.locator(".ant-select-dropdown:visible").getByText("All Proxy Models").click(); + await page.keyboard.press("Escape"); + + // Submit — click the submit button inside the dialog (not the header button) + await dialog.locator("button[type='submit']").click(); + + // Verify success notification + await expect(page.getByText("Team created").first()).toBeVisible({ timeout: 10_000 }); + }); + + test("Invite a user to a team", async ({ page }) => { + await navigateToPage(page, Page.Teams); + await dismissFeedbackPopup(page); + + await clickTeamId(page, E2E_TEAM_CRUD_ID); + + await page.getByRole("tab", { name: "Members" }).click(); + await page.getByRole("button", { name: /Add Member/i }).click(); + + // Wait for Add Team Member modal + const modal = page.locator(".ant-modal:visible"); + await expect(modal).toBeVisible({ timeout: 5_000 }); + + // The email field is a Select — type to search, then select from dropdown + await modal.locator(".ant-select").first().click(); + await page.keyboard.type("invitable@test.local"); + + // Wait for the option to appear, then select via keyboard (avoids viewport issues) + const emailOption = page.getByRole("option", { name: "invitable@test.local" }).first(); + await expect(emailOption).toBeAttached({ timeout: 10_000 }); + // Use keyboard to select the highlighted option + await page.keyboard.press("Enter"); + + // Submit + await modal.getByRole("button", { name: /Add Member/i }).click(); + + await expect(page.getByText(/member.*added|success/i).first()).toBeVisible({ timeout: 10_000 }); + }); + + test("Edit team member for team proxy admin does not belong to", async ({ page }) => { + await navigateToPage(page, Page.Teams); + await dismissFeedbackPopup(page); + + await clickTeamId(page, E2E_TEAM_NO_ADMIN_ID); + + await page.getByRole("tab", { name: "Members" }).click(); + + await page.getByTestId("edit-member").first().click(); + + const modal = page.locator(".ant-modal:visible"); + await expect(modal).toBeVisible({ timeout: 5_000 }); + await modal.getByRole("button", { name: /Save Changes/i }).click(); + + await expect(page.getByText(/updated|success/i).first()).toBeVisible({ timeout: 10_000 }); + }); + + test("Delete a team", async ({ page }) => { + await navigateToPage(page, Page.Teams); + await dismissFeedbackPopup(page); + + const teamRow = page.locator("tr", { hasText: E2E_TEAM_DELETE_ALIAS }).first(); + await expect(teamRow).toBeVisible({ timeout: 10_000 }); + await teamRow.locator("svg, img").last().click(); + + const modal = page.locator(".ant-modal:visible"); + await expect(modal).toBeVisible({ timeout: 5_000 }); + await modal.locator("input").fill(E2E_TEAM_DELETE_ALIAS); + await modal.getByRole("button", { name: /Force Delete|Delete/i }).click(); + + await expect(teamRow).not.toBeVisible({ timeout: 10_000 }); + }); + + test("Team in org - edit team member", async ({ page }) => { + await navigateToPage(page, Page.Teams); + await dismissFeedbackPopup(page); + + await clickTeamId(page, E2E_TEAM_ORG_ID); + + await page.getByRole("tab", { name: "Members" }).click(); + + await page.getByTestId("edit-member").first().click(); + + const modal = page.locator(".ant-modal:visible"); + await expect(modal).toBeVisible({ timeout: 5_000 }); + await modal.getByRole("button", { name: /Save Changes/i }).click(); + + await expect(page.getByText(/updated|success/i).first()).toBeVisible({ timeout: 10_000 }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsTable/TeamsTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsTable/TeamsTable.tsx index 4533d99b4a..f881065d4a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsTable/TeamsTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsTable/TeamsTable.tsx @@ -80,6 +80,7 @@ const TeamsTable = ({ size="xs" variant="light" className="font-mono text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs font-normal px-2 py-0.5 text-left overflow-hidden truncate max-w-[200px]" + data-testid="team-id-cell" onClick={() => { // Add click handler setSelectedTeamId(team.team_id); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/teams/components/modals/CreateTeamModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/teams/components/modals/CreateTeamModal.tsx index 0aa42b69a0..ecaa3c08a4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/teams/components/modals/CreateTeamModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/teams/components/modals/CreateTeamModal.tsx @@ -312,7 +312,7 @@ const CreateTeamModal = ({ }, ]} > - + - + All Proxy Models @@ -716,7 +716,7 @@ const CreateTeamModal = ({
- Create Team + Create Team
diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index 76dd6abbe6..8349e271b8 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -695,6 +695,7 @@ const Teams: React.FC = ({ className="text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs cursor-pointer" style={{ fontSize: 14, padding: "1px 8px" }} onClick={() => setSelectedTeamId(record.team_id)} + data-testid="team-id-cell" > {id} @@ -898,6 +899,7 @@ const Teams: React.FC = ({ icon={} onClick={() => setIsTeamModalVisible(true)} style={{ marginTop: 16 }} + data-testid="create-team-button" > Create Team @@ -1041,7 +1043,7 @@ const Teams: React.FC = ({ {canCreateOrManageTeams(userRole, userID, organizations) && ( - )} @@ -1078,7 +1080,7 @@ const Teams: React.FC = ({ }, ]} > - + {(() => { const adminOrgs = getAdminOrganizations(userRole, userID, organizations); @@ -1567,7 +1569,7 @@ const Teams: React.FC = ({
- +
diff --git a/ui/litellm-dashboard/src/components/common_components/team_dropdown.tsx b/ui/litellm-dashboard/src/components/common_components/team_dropdown.tsx index 844bbfc3eb..8bdde4771f 100644 --- a/ui/litellm-dashboard/src/components/common_components/team_dropdown.tsx +++ b/ui/litellm-dashboard/src/components/common_components/team_dropdown.tsx @@ -96,6 +96,7 @@ const TeamDropdown: React.FC = ({ onPopupScroll={handlePopupScroll} loading={isLoading} notFoundContent={isLoading ? : "No teams found"} + data-testid="team-dropdown" popupRender={(menu) => ( <> {menu} diff --git a/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx b/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx index 4427f78bb8..866d7cbec7 100644 --- a/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx +++ b/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx @@ -150,6 +150,7 @@ const UserSearchModal: React.FC = ({ options={selectedField === "user_email" ? userOptions : []} loading={loading} allowClear + data-testid="member-email-search" /> diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 76888262e5..753b6d5fcf 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -666,7 +666,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp return (
{userRole && rolesWithWriteAccess.includes(userRole) && ( - )} From a8f4f464ceb4602e6b2c75e899444745649e15a8 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 8 Apr 2026 12:40:41 -0700 Subject: [PATCH 2/5] [Fix] Add missing test fixtures and address review feedback - Add constants.ts with all required exports (key aliases, team IDs) - Add fixtures/users.ts with all role definitions and storage paths - Add fixtures/seed.sql for deterministic test database seeding - Remove Firefox project from playwright config (only Chromium installed) - Remove unused variable in teams.spec.ts - Rename CircleCI job to e2e_ui_testing --- .circleci/config.yml | 2 +- ui/litellm-dashboard/e2e_tests/constants.ts | 24 +++++- .../e2e_tests/fixtures/seed.sql | 84 +++++++++++++++++++ .../e2e_tests/fixtures/users.ts | 38 +++++++-- .../e2e_tests/playwright.config.ts | 5 -- .../e2e_tests/tests/proxy-admin/teams.spec.ts | 2 - 6 files changed, 138 insertions(+), 17 deletions(-) create mode 100644 ui/litellm-dashboard/e2e_tests/fixtures/seed.sql diff --git a/.circleci/config.yml b/.circleci/config.yml index a9d395e768..5f1f196b44 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3074,7 +3074,7 @@ jobs: CI=true npm run test -- --run \ --pool forks --poolOptions.forks.maxForks=8 - ui_e2e_tests: + e2e_ui_testing: docker: - image: cimg/python:3.12-browsers auth: diff --git a/ui/litellm-dashboard/e2e_tests/constants.ts b/ui/litellm-dashboard/e2e_tests/constants.ts index 58b56af0a2..dbc73432f6 100644 --- a/ui/litellm-dashboard/e2e_tests/constants.ts +++ b/ui/litellm-dashboard/e2e_tests/constants.ts @@ -1,6 +1,22 @@ +// Storage state paths for each role export const ADMIN_STORAGE_PATH = "admin.storageState.json"; +export const ADMIN_VIEWER_STORAGE_PATH = "adminViewer.storageState.json"; +export const INTERNAL_USER_STORAGE_PATH = "internalUser.storageState.json"; +export const INTERNAL_VIEWER_STORAGE_PATH = "internalViewer.storageState.json"; +export const TEAM_ADMIN_STORAGE_PATH = "teamAdmin.storageState.json"; -export const E2E_UPDATE_LIMITS_KEY_ID_PREFIX = "102c"; -export const E2E_DELETE_KEY_ID_PREFIX = "94a5"; -export const E2E_DELETE_KEY_NAME = "e2eDeleteKey"; -export const E2E_REGENERATE_KEY_ID_PREFIX = "593a"; +// Key aliases for seeded test keys (match seed.sql) +export const E2E_UPDATE_LIMITS_KEY_ALIAS = "e2eUpdateLimitsKey"; +export const E2E_DELETE_KEY_ALIAS = "e2eDeleteKey"; +export const E2E_REGENERATE_KEY_ALIAS = "e2eRegenerateKey"; +export const E2E_INTERNAL_USER_KEY_ALIAS = "e2eInternalUserKey"; +export const E2E_VIEWER_KEY_ALIAS = "e2eViewerKey"; + +// Team identifiers (match seed.sql) +export const E2E_TEAM_CRUD_ID = "e2e-team-crud"; +export const E2E_TEAM_CRUD_ALIAS = "E2E Team CRUD"; +export const E2E_TEAM_DELETE_ID = "e2e-team-delete"; +export const E2E_TEAM_DELETE_ALIAS = "E2E Team Delete"; +export const E2E_TEAM_ORG_ID = "e2e-team-org"; +export const E2E_TEAM_NO_ADMIN_ID = "e2e-team-no-admin"; +export const E2E_TEAM_NO_ADMIN_ALIAS = "E2E Team No Admin"; diff --git a/ui/litellm-dashboard/e2e_tests/fixtures/seed.sql b/ui/litellm-dashboard/e2e_tests/fixtures/seed.sql new file mode 100644 index 0000000000..91312e66ce --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/fixtures/seed.sql @@ -0,0 +1,84 @@ +-- E2E Test Seed Data +-- Idempotent: deletes all e2e-* rows then re-inserts deterministic data. + +-- 1. Clean up in dependency order +DELETE FROM "LiteLLM_TeamMembership" WHERE "user_id" LIKE 'e2e-%'; +DELETE FROM "LiteLLM_VerificationToken" WHERE token LIKE 'e2e-%'; +DELETE FROM "LiteLLM_TeamTable" WHERE "team_id" LIKE 'e2e-%'; +DELETE FROM "LiteLLM_OrganizationTable" WHERE "organization_id" LIKE 'e2e-%'; +DELETE FROM "LiteLLM_UserTable" WHERE "user_id" LIKE 'e2e-%'; +DELETE FROM "LiteLLM_BudgetTable" WHERE "budget_id" LIKE 'e2e-%'; + +-- 2. Budget (created_by and updated_by are NOT NULL) +INSERT INTO "LiteLLM_BudgetTable" ("budget_id", "max_budget", "created_by", "updated_by") +VALUES ('e2e-budget-org', 1000, 'e2e-proxy-admin', 'e2e-proxy-admin'); + +-- 3. Organization (created_by and updated_by are NOT NULL) +INSERT INTO "LiteLLM_OrganizationTable" ( + "organization_id", "organization_alias", "budget_id", + "metadata", "models", "spend", "model_spend", + "created_by", "updated_by" +) VALUES ( + 'e2e-org-main', 'E2E Organization', 'e2e-budget-org', + '{}'::jsonb, ARRAY[]::text[], 0.0, '{}'::jsonb, + 'e2e-proxy-admin', 'e2e-proxy-admin' +); + +-- 4. Users (password hash is scrypt of "test") +INSERT INTO "LiteLLM_UserTable" ("user_id", "user_email", "user_role", "teams", "password") +VALUES + ('e2e-proxy-admin', 'admin@test.local', 'proxy_admin', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), + ('e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), + ('e2e-internal-user', 'internal@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-org"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), + ('e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), + ('e2e-team-admin', 'teamadmin@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-delete"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), + ('e2e-invitable-user', 'invitable@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), + ('e2e-removable-member', 'removable@test.local', 'internal_user', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'); + +-- 5. Teams (members_with_roles is required JSON) +INSERT INTO "LiteLLM_TeamTable" ( + "team_id", "team_alias", "organization_id", "admins", "members", + "members_with_roles", "metadata", "models", "spend", "model_spend", "model_max_budget", "blocked" +) VALUES + ('e2e-team-crud', 'E2E Team CRUD', NULL, + '{"e2e-team-admin"}', + '{"e2e-team-admin","e2e-internal-user","e2e-internal-viewer","e2e-removable-member"}', + '[{"role":"admin","user_id":"e2e-team-admin"},{"role":"user","user_id":"e2e-internal-user"},{"role":"user","user_id":"e2e-internal-viewer"},{"role":"user","user_id":"e2e-removable-member"}]'::jsonb, + '{}'::jsonb, '{"fake-openai-gpt-4","fake-anthropic-claude"}', 0.0, '{}'::jsonb, '{}'::jsonb, false), + + ('e2e-team-delete', 'E2E Team Delete', NULL, + '{"e2e-team-admin"}', '{"e2e-team-admin"}', + '[{"role":"admin","user_id":"e2e-team-admin"}]'::jsonb, + '{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false), + + ('e2e-team-org', 'E2E Team In Org', 'e2e-org-main', + '{}', '{"e2e-internal-user"}', + '[{"role":"user","user_id":"e2e-internal-user"}]'::jsonb, + '{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false), + + ('e2e-team-no-admin', 'E2E Team No Admin', NULL, + '{}', '{"e2e-invitable-user"}', + '[{"role":"user","user_id":"e2e-invitable-user"}]'::jsonb, + '{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false); + +-- 6. Team Memberships (only user_id, team_id, spend — no created_at/updated_at) +INSERT INTO "LiteLLM_TeamMembership" ("user_id", "team_id", "spend") +VALUES + ('e2e-team-admin', 'e2e-team-crud', 0.0), + ('e2e-internal-user', 'e2e-team-crud', 0.0), + ('e2e-internal-viewer', 'e2e-team-crud', 0.0), + ('e2e-removable-member', 'e2e-team-crud', 0.0), + ('e2e-team-admin', 'e2e-team-delete', 0.0), + ('e2e-internal-user', 'e2e-team-org', 0.0), + ('e2e-invitable-user', 'e2e-team-no-admin', 0.0); + +-- 7. Verification Tokens (API Keys) +INSERT INTO "LiteLLM_VerificationToken" ( + "token", "key_name", "key_alias", "user_id", "team_id", + "models", "spend", "max_budget", "expires", "metadata" +) VALUES + ('e2e-key-update-limits', 'sk-e2e-update', 'e2eUpdateLimitsKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), + ('e2e-key-delete', 'sk-e2e-delete', 'e2eDeleteKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), + ('e2e-key-regenerate', 'sk-e2e-regen', 'e2eRegenerateKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), + ('e2e-key-internal-user', 'sk-e2e-internal', 'e2eInternalUserKey', 'e2e-internal-user', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), + ('e2e-key-viewer', 'sk-e2e-viewer', 'e2eViewerKey', 'e2e-internal-viewer', NULL, '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb); diff --git a/ui/litellm-dashboard/e2e_tests/fixtures/users.ts b/ui/litellm-dashboard/e2e_tests/fixtures/users.ts index d1f1eab00e..7d6d356cef 100644 --- a/ui/litellm-dashboard/e2e_tests/fixtures/users.ts +++ b/ui/litellm-dashboard/e2e_tests/fixtures/users.ts @@ -1,10 +1,38 @@ -import { Role } from "./roles"; +export enum Role { + ProxyAdmin = "proxy_admin", + ProxyAdminViewer = "proxy_admin_viewer", + InternalUser = "internal_user", + InternalUserViewer = "internal_user_viewer", + TeamAdmin = "team_admin", +} -const isCI = !!process.env.CI; - -export const users = { +export const users: Record = { [Role.ProxyAdmin]: { email: "admin", - password: isCI ? "gm" : "sk-1234", + password: process.env.LITELLM_MASTER_KEY || "sk-1234", + }, + [Role.ProxyAdminViewer]: { + email: "adminviewer@test.local", + password: "test", + }, + [Role.InternalUser]: { + email: "internal@test.local", + password: "test", + }, + [Role.InternalUserViewer]: { + email: "viewer@test.local", + password: "test", + }, + [Role.TeamAdmin]: { + email: "teamadmin@test.local", + password: "test", }, }; + +export const STORAGE_PATHS: Record = { + [Role.ProxyAdmin]: "admin.storageState.json", + [Role.ProxyAdminViewer]: "adminViewer.storageState.json", + [Role.InternalUser]: "internalUser.storageState.json", + [Role.InternalUserViewer]: "internalViewer.storageState.json", + [Role.TeamAdmin]: "teamAdmin.storageState.json", +}; diff --git a/ui/litellm-dashboard/e2e_tests/playwright.config.ts b/ui/litellm-dashboard/e2e_tests/playwright.config.ts index fd18a1d9bd..ec4d3a6ddb 100644 --- a/ui/litellm-dashboard/e2e_tests/playwright.config.ts +++ b/ui/litellm-dashboard/e2e_tests/playwright.config.ts @@ -36,11 +36,6 @@ export default defineConfig({ name: "chromium", use: { ...devices["Desktop Chrome"] }, }, - - { - name: "firefox", - use: { ...devices["Desktop Firefox"] }, - }, ], /* Timeout settings */ diff --git a/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts index 19c33a2cd7..a1864b22a4 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts @@ -15,8 +15,6 @@ import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; * clickable span (OldTeams Typography.Text). */ async function clickTeamId(page: import("@playwright/test").Page, teamId: string) { - const idPrefix = teamId.slice(0, 7); - // The team ID is either a Button or a clickable span containing the first 7 chars const cell = page.locator("td").filter({ hasText: teamId }).first(); await expect(cell).toBeVisible({ timeout: 10_000 }); await cell.click(); From ac9ebdf4d8d90b3f2d9fed2faedca1925ff76ccf Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 8 Apr 2026 13:17:45 -0700 Subject: [PATCH 3/5] [Fix] Rename CI job to e2e_ui_testing and remove duplicate old job definition --- .circleci/config.yml | 76 +------------------------------------------- 1 file changed, 1 insertion(+), 75 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5f1f196b44..cf2c417661 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3206,80 +3206,6 @@ jobs: paths: - litellm-docker-database.tar.zst - e2e_ui_testing: - machine: - image: ubuntu-2204:2023.10.1 - resource_class: large - working_directory: ~/project - parameters: - browser: - type: string - steps: - - checkout - - setup_google_dns - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Install Dependencies - command: | - npm install -D @playwright/test - - run: - name: Install Playwright Browsers - command: | - npx playwright install - - run: - name: Run Docker container - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \ - -e LITELLM_MASTER_KEY="sk-1234" \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e UI_USERNAME="admin" \ - -e UI_PASSWORD="gm" \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - --name litellm-docker-database-<< parameters.browser >> \ - -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start outputting logs - command: docker logs -f litellm-docker-database-<< parameters.browser >> - background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m - - run: - name: Run Playwright Tests - command: | - npx playwright test \ - --project << parameters.browser >> \ - --config ui/litellm-dashboard/e2e_tests/playwright.config.ts \ - --reporter=html \ - --output=test-results - no_output_timeout: 15m - - store_artifacts: - path: test-results - destination: playwright-results - - - store_artifacts: - path: playwright-report - destination: playwright-report prisma_schema_sync: machine: @@ -3508,7 +3434,7 @@ workflows: only: - main - /litellm_.*/ - - ui_e2e_tests: + - e2e_ui_testing: filters: branches: only: From 4ee7d42981a2b3b5bcae3b6fd33135bc66aa1597 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 8 Apr 2026 13:24:52 -0700 Subject: [PATCH 4/5] [Fix] Restructure HTML files after UI build so extensionless routes work in CI --- .circleci/config.yml | 4 ++++ ui/litellm-dashboard/e2e_tests/run_e2e.sh | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf2c417661..dd7635592c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3127,6 +3127,10 @@ jobs: cd ui/litellm-dashboard npm run build cp -r out/ ../../litellm/proxy/_experimental/out/ + # Restructure HTML so extensionless routes work (login.html -> login/index.html) + find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do + d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" + done - run: name: Wait for PostgreSQL command: dockerize -wait tcp://localhost:5432 -timeout 30s diff --git a/ui/litellm-dashboard/e2e_tests/run_e2e.sh b/ui/litellm-dashboard/e2e_tests/run_e2e.sh index 9e97914658..4e3a47edfb 100755 --- a/ui/litellm-dashboard/e2e_tests/run_e2e.sh +++ b/ui/litellm-dashboard/e2e_tests/run_e2e.sh @@ -103,7 +103,16 @@ npm install --silent 2>/dev/null || true npm run build # Copy the fresh build to the proxy's static UI directory cp -r "$DASHBOARD_DIR/out/" "$REPO_ROOT/litellm/proxy/_experimental/out/" -echo "UI build copied to proxy static directory" + +# Restructure HTML files so extensionless routes work (e.g. /ui/login) +# Next.js export produces login.html; the proxy expects login/index.html +find "$REPO_ROOT/litellm/proxy/_experimental/out" -name '*.html' ! -name 'index.html' | while read -r htmlfile; do + target_dir="${htmlfile%.html}" + target_path="$target_dir/index.html" + mkdir -p "$target_dir" + mv "$htmlfile" "$target_path" +done +echo "UI build copied and restructured" # --- Python environment --- echo "=== Setting up Python environment ===" From 467dbc4a3c1b74473b245da21ce3e9faeb82e534 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 8 Apr 2026 13:32:37 -0700 Subject: [PATCH 5/5] [Fix] Remove old broken key tests superseded by proxy-admin/keys.spec.ts --- .../e2e_tests/tests/keys/createKey.spec.ts | 22 --------------- .../e2e_tests/tests/keys/deleteKey.spec.ts | 25 ----------------- .../tests/keys/regenerateKey.spec.ts | 21 --------------- .../tests/keys/updateKeyLimits.spec.ts | 27 ------------------- 4 files changed, 95 deletions(-) delete mode 100644 ui/litellm-dashboard/e2e_tests/tests/keys/createKey.spec.ts delete mode 100644 ui/litellm-dashboard/e2e_tests/tests/keys/deleteKey.spec.ts delete mode 100644 ui/litellm-dashboard/e2e_tests/tests/keys/regenerateKey.spec.ts delete mode 100644 ui/litellm-dashboard/e2e_tests/tests/keys/updateKeyLimits.spec.ts diff --git a/ui/litellm-dashboard/e2e_tests/tests/keys/createKey.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/keys/createKey.spec.ts deleted file mode 100644 index 682d1a1b45..0000000000 --- a/ui/litellm-dashboard/e2e_tests/tests/keys/createKey.spec.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { ADMIN_STORAGE_PATH } from "../../constants"; -import { Page } from "../../fixtures/pages"; -import { navigateToPage } from "../../helpers/navigation"; - -test.describe("Create Key", () => { - test.use({ storageState: ADMIN_STORAGE_PATH }); - - test("Able to create a key with all team models", async ({ page }) => { - await navigateToPage(page, Page.ApiKeys); - await expect(page.getByRole("button", { name: "Next" })).toBeVisible(); - await page.getByRole("button", { name: "+ Create New Key" }).click(); - await page.getByTestId("base-input").click(); - await page.getByTestId("base-input").fill("e2eUITestingCreateKeyAllTeamModels"); - await page.locator(".ant-select-selection-overflow").click(); - await page.getByText("All Team Models").click(); - await page.getByRole("combobox", { name: /models/i }).press("Escape"); - await page.getByRole("button", { name: "Create Key" }).click(); - await page.keyboard.press("Escape"); - await expect(page.getByText("e2eUITestingCreateKeyAllTeamModels")).toBeVisible(); - }); -}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/keys/deleteKey.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/keys/deleteKey.spec.ts deleted file mode 100644 index a584131625..0000000000 --- a/ui/litellm-dashboard/e2e_tests/tests/keys/deleteKey.spec.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { ADMIN_STORAGE_PATH, E2E_DELETE_KEY_ID_PREFIX, E2E_DELETE_KEY_NAME } from "../../constants"; -import { Page } from "../../fixtures/pages"; -import { navigateToPage } from "../../helpers/navigation"; - -test.describe("Delete Key", () => { - test.use({ storageState: ADMIN_STORAGE_PATH }); - - test("Able to delete a key", async ({ page }) => { - await navigateToPage(page, Page.ApiKeys); - await expect(page.getByRole("button", { name: "Next" })).toBeVisible(); - await page - .locator("button", { - hasText: E2E_DELETE_KEY_ID_PREFIX, - }) - .click(); - await page.getByRole("button", { name: "Delete Key" }).click(); - await page.getByRole("textbox", { name: E2E_DELETE_KEY_NAME }).click(); - await page.getByRole("textbox", { name: E2E_DELETE_KEY_NAME }).fill(E2E_DELETE_KEY_NAME); - const deleteButton = page.getByRole("button", { name: "Delete", exact: true }); - await expect(deleteButton).toBeEnabled(); - await deleteButton.click(); - await expect(page.getByText("Key deleted successfully")).toBeVisible(); - }); -}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/keys/regenerateKey.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/keys/regenerateKey.spec.ts deleted file mode 100644 index 0188a4f81c..0000000000 --- a/ui/litellm-dashboard/e2e_tests/tests/keys/regenerateKey.spec.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { ADMIN_STORAGE_PATH, E2E_REGENERATE_KEY_ID_PREFIX } from "../../constants"; -import { Page } from "../../fixtures/pages"; -import { navigateToPage } from "../../helpers/navigation"; - -test.describe("Regenerate Key", () => { - test.use({ storageState: ADMIN_STORAGE_PATH }); - - test("Able to regenerate a key", async ({ page }) => { - await navigateToPage(page, Page.ApiKeys); - await expect(page.getByRole("button", { name: "Next" })).toBeVisible(); - await page - .locator("button", { - hasText: E2E_REGENERATE_KEY_ID_PREFIX, - }) - .click(); - await page.getByRole("button", { name: "Regenerate Key" }).click(); - await page.getByRole("button", { name: "Regenerate", exact: true }).click(); - await expect(page.getByText("Virtual Key regenerated")).toBeVisible(); - }); -}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/keys/updateKeyLimits.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/keys/updateKeyLimits.spec.ts deleted file mode 100644 index 6cae36272a..0000000000 --- a/ui/litellm-dashboard/e2e_tests/tests/keys/updateKeyLimits.spec.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { ADMIN_STORAGE_PATH, E2E_UPDATE_LIMITS_KEY_ID_PREFIX } from "../../constants"; -import { Page } from "../../fixtures/pages"; -import { navigateToPage } from "../../helpers/navigation"; - -test.describe("Update Key TPM and RPM Limits", () => { - test.use({ storageState: ADMIN_STORAGE_PATH }); - - test("Able to update a key's TPM and RPM limits", async ({ page }) => { - await navigateToPage(page, Page.ApiKeys); - await expect(page.getByRole("button", { name: "Next" })).toBeVisible(); - await page - .locator("button", { - hasText: E2E_UPDATE_LIMITS_KEY_ID_PREFIX, - }) - .click(); - await page.getByRole("tab", { name: "Settings" }).click(); - await page.getByRole("button", { name: "Edit Settings" }).click(); - await page.getByRole("spinbutton", { name: "TPM Limit" }).click(); - await page.getByRole("spinbutton", { name: "TPM Limit" }).fill("123"); - await page.getByRole("spinbutton", { name: "RPM Limit" }).click(); - await page.getByRole("spinbutton", { name: "RPM Limit" }).fill("456"); - await page.getByRole("button", { name: "Save Changes" }).click(); - await expect(page.getByRole("paragraph").filter({ hasText: "TPM: 123" })).toBeVisible(); - await expect(page.getByRole("paragraph").filter({ hasText: "RPM: 456" })).toBeVisible(); - }); -});