[Feature] E2E UI tests: proxy-admin team and key management with CI integration
Add Playwright E2E tests covering proxy admin team and key management workflows, with a self-contained test runner and CircleCI integration. Tests cover: create team, invite user, edit/delete team members, create key in team, regenerate key, update TPM/RPM limits, delete key, and verify internal user keys are visible. Infrastructure: run_e2e.sh builds the UI from source before starting the proxy, ensuring tests always run against the latest UI changes. Added data-testid attributes to key UI components for reliable selectors.
This commit is contained in:
parent
2dac54b732
commit
d09d98a70a
@ -3074,6 +3074,113 @@ jobs:
|
||||
CI=true npm run test -- --run \
|
||||
--pool forks --poolOptions.forks.maxForks=8
|
||||
|
||||
ui_e2e_tests:
|
||||
docker:
|
||||
- image: cimg/python:3.12-browsers
|
||||
auth:
|
||||
username: ${DOCKERHUB_USERNAME}
|
||||
password: ${DOCKERHUB_PASSWORD}
|
||||
- image: cimg/postgres:16.0
|
||||
environment:
|
||||
POSTGRES_USER: e2euser
|
||||
POSTGRES_PASSWORD: e2epassword
|
||||
POSTGRES_DB: litellm_e2e
|
||||
resource_class: large
|
||||
working_directory: ~/project
|
||||
environment:
|
||||
DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
|
||||
CI: "true"
|
||||
steps:
|
||||
- checkout
|
||||
- setup_google_dns
|
||||
- restore_cache:
|
||||
keys:
|
||||
- ui-e2e-py-deps-v1-{{ checksum "requirements.txt" }}
|
||||
- run:
|
||||
name: Install Python dependencies
|
||||
command: |
|
||||
python -m pip install --upgrade pip uv
|
||||
uv pip install --system -r requirements.txt
|
||||
pip install "prisma==0.11.0"
|
||||
prisma generate --schema litellm/proxy/schema.prisma
|
||||
- save_cache:
|
||||
key: ui-e2e-py-deps-v1-{{ checksum "requirements.txt" }}
|
||||
paths:
|
||||
- ~/.local/lib
|
||||
- ~/.local/bin
|
||||
- restore_cache:
|
||||
keys:
|
||||
- ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
||||
- run:
|
||||
name: Install Node dependencies and Playwright
|
||||
command: |
|
||||
cd ui/litellm-dashboard
|
||||
npm ci
|
||||
npx playwright install chromium --with-deps
|
||||
- save_cache:
|
||||
key: ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
||||
paths:
|
||||
- ui/litellm-dashboard/node_modules
|
||||
- run:
|
||||
name: Build UI from source
|
||||
command: |
|
||||
cd ui/litellm-dashboard
|
||||
npm run build
|
||||
cp -r out/ ../../litellm/proxy/_experimental/out/
|
||||
- run:
|
||||
name: Wait for PostgreSQL
|
||||
command: dockerize -wait tcp://localhost:5432 -timeout 30s
|
||||
- run:
|
||||
name: Push Prisma schema
|
||||
command: prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
|
||||
- run:
|
||||
name: Seed database
|
||||
command: |
|
||||
PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
|
||||
-f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
|
||||
- run:
|
||||
name: Start mock LLM server
|
||||
command: python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
|
||||
background: true
|
||||
- run:
|
||||
name: Start LiteLLM proxy
|
||||
environment:
|
||||
LITELLM_MASTER_KEY: "sk-1234"
|
||||
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
|
||||
DISABLE_SCHEMA_UPDATE: "true"
|
||||
SERVER_ROOT_PATH: ""
|
||||
PROXY_LOGOUT_URL: ""
|
||||
command: |
|
||||
python -m litellm.proxy.proxy_cli \
|
||||
--config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
|
||||
--port 4000
|
||||
background: true
|
||||
- run:
|
||||
name: Wait for proxy to be ready
|
||||
command: |
|
||||
for i in $(seq 1 60); do
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true)
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
echo "Proxy is ready"
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "Proxy failed to start"
|
||||
exit 1
|
||||
- run:
|
||||
name: Run Playwright E2E tests
|
||||
command: |
|
||||
cd ui/litellm-dashboard
|
||||
npx playwright test --config e2e_tests/playwright.config.ts
|
||||
no_output_timeout: 10m
|
||||
- store_artifacts:
|
||||
path: ui/litellm-dashboard/test-results
|
||||
destination: e2e-test-results
|
||||
- store_artifacts:
|
||||
path: ui/litellm-dashboard/playwright-report
|
||||
destination: e2e-playwright-report
|
||||
|
||||
build_docker_database_image:
|
||||
machine:
|
||||
image: ubuntu-2204:2024.04.1
|
||||
@ -3401,32 +3508,12 @@ workflows:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
# - e2e_ui_testing:
|
||||
# name: e2e_ui_testing_chromium
|
||||
# browser: chromium
|
||||
# context: e2e_ui_tests
|
||||
# requires:
|
||||
# - ui_build
|
||||
# - build_docker_database_image
|
||||
# - prisma_schema_sync
|
||||
# filters:
|
||||
# branches:
|
||||
# only:
|
||||
# - main
|
||||
# - /litellm_.*/
|
||||
# - e2e_ui_testing:
|
||||
# name: e2e_ui_testing_firefox
|
||||
# browser: firefox
|
||||
# context: e2e_ui_tests
|
||||
# requires:
|
||||
# - ui_build
|
||||
# - build_docker_database_image
|
||||
# - prisma_schema_sync
|
||||
# filters:
|
||||
# branches:
|
||||
# only:
|
||||
# - main
|
||||
# - /litellm_.*/
|
||||
- ui_e2e_tests:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- build_and_test:
|
||||
requires:
|
||||
- build_docker_database_image
|
||||
|
||||
16
ui/litellm-dashboard/e2e_tests/fixtures/config.yml
Normal file
16
ui/litellm-dashboard/e2e_tests/fixtures/config.yml
Normal file
@ -0,0 +1,16 @@
|
||||
model_list:
|
||||
- model_name: fake-openai-gpt-4
|
||||
litellm_params:
|
||||
model: openai/fake-gpt-4
|
||||
api_base: os.environ/MOCK_LLM_URL
|
||||
api_key: fake-key
|
||||
- model_name: fake-anthropic-claude
|
||||
litellm_params:
|
||||
model: openai/fake-claude
|
||||
api_base: os.environ/MOCK_LLM_URL
|
||||
api_key: fake-key
|
||||
|
||||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
database_url: os.environ/DATABASE_URL
|
||||
store_prompts_in_spend_logs: true
|
||||
@ -0,0 +1,120 @@
|
||||
"""
|
||||
Mock LLM server for UI e2e tests.
|
||||
Responds to OpenAI-format endpoints with canned responses.
|
||||
"""
|
||||
|
||||
import time
|
||||
import json
|
||||
import uuid
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
|
||||
app = FastAPI(title="Mock LLM Server")
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/v1/models")
|
||||
@app.get("/models")
|
||||
async def list_models():
|
||||
return {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id": "fake-gpt-4", "object": "model", "owned_by": "mock"},
|
||||
{"id": "fake-claude", "object": "model", "owned_by": "mock"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@app.post("/v1/chat/completions")
|
||||
@app.post("/chat/completions")
|
||||
async def chat_completions(request: Request):
|
||||
body = await request.json()
|
||||
model = body.get("model", "mock-model")
|
||||
stream = body.get("stream", False)
|
||||
|
||||
response_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
|
||||
created = int(time.time())
|
||||
|
||||
if stream:
|
||||
|
||||
async def stream_generator():
|
||||
chunk = {
|
||||
"id": response_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": created,
|
||||
"model": model,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"role": "assistant",
|
||||
"content": "This is a mock response.",
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
yield f"data: {json.dumps(chunk)}\n\n"
|
||||
|
||||
done_chunk = {
|
||||
"id": response_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": created,
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
|
||||
}
|
||||
yield f"data: {json.dumps(done_chunk)}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
return StreamingResponse(stream_generator(), media_type="text/event-stream")
|
||||
|
||||
return {
|
||||
"id": response_id,
|
||||
"object": "chat.completion",
|
||||
"created": created,
|
||||
"model": model,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "This is a mock response."},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18},
|
||||
}
|
||||
|
||||
|
||||
@app.post("/v1/embeddings")
|
||||
@app.post("/embeddings")
|
||||
async def embeddings(request: Request):
|
||||
body = await request.json()
|
||||
inputs = body.get("input", [""])
|
||||
if isinstance(inputs, str):
|
||||
inputs = [inputs]
|
||||
return {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"object": "embedding", "index": i, "embedding": [0.0] * 1536}
|
||||
for i in range(len(inputs))
|
||||
],
|
||||
"model": body.get("model", "mock-embedding"),
|
||||
"usage": {"prompt_tokens": 5, "total_tokens": 5},
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run(app, host="127.0.0.1", port=8090)
|
||||
@ -1,17 +1,40 @@
|
||||
import { chromium } from "@playwright/test";
|
||||
import { users } from "./fixtures/users";
|
||||
import { Role } from "./fixtures/roles";
|
||||
import { chromium, expect } from "@playwright/test";
|
||||
import { users, Role, STORAGE_PATHS } from "./fixtures/users";
|
||||
import * as fs from "fs";
|
||||
|
||||
async function globalSetup() {
|
||||
const browser = await chromium.launch();
|
||||
const page = await browser.newPage();
|
||||
await page.goto("http://localhost:4000/ui/login");
|
||||
await page.getByPlaceholder("Enter your username").fill(users[Role.ProxyAdmin].email);
|
||||
await page.getByPlaceholder("Enter your password").fill(users[Role.ProxyAdmin].password);
|
||||
const loginButton = page.getByRole("button", { name: "Login", exact: true });
|
||||
await loginButton.click();
|
||||
await page.waitForSelector("text=Virtual Keys");
|
||||
await page.context().storageState({ path: "admin.storageState.json" });
|
||||
|
||||
for (const role of Object.values(Role)) {
|
||||
const { email, password } = users[role];
|
||||
const storagePath = STORAGE_PATHS[role];
|
||||
const page = await browser.newPage();
|
||||
try {
|
||||
await page.goto("http://localhost:4000/ui/login");
|
||||
await page.getByPlaceholder("Enter your username").fill(email);
|
||||
await page.getByPlaceholder("Enter your password").fill(password);
|
||||
await page.getByRole("button", { name: "Login", exact: true }).click();
|
||||
await page.waitForURL(
|
||||
(url) => url.pathname.startsWith("/ui") && !url.pathname.includes("/login"),
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
await expect(page.locator("a", { hasText: "Virtual Keys" })).toBeVisible({ timeout: 30_000 });
|
||||
// Dismiss feedback popup if present
|
||||
const dismiss = page.getByText("Don't ask me again");
|
||||
if (await dismiss.isVisible({ timeout: 1_500 }).catch(() => false)) {
|
||||
await dismiss.click();
|
||||
}
|
||||
await page.context().storageState({ path: storagePath });
|
||||
} catch (e) {
|
||||
fs.mkdirSync("test-results", { recursive: true });
|
||||
await page.screenshot({ path: `test-results/global-setup-${role}-failure.png`, fullPage: true });
|
||||
console.error(`Global setup failed for role ${role}. Screenshot saved. URL: ${page.url()}`);
|
||||
throw e;
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
|
||||
@ -1,12 +1,25 @@
|
||||
import { Page } from "../fixtures/pages";
|
||||
import { Page as PlaywrightPage } from "@playwright/test";
|
||||
import { Page as PlaywrightPage, expect } from "@playwright/test";
|
||||
|
||||
/**
|
||||
* Navigates to a specific page using the page query parameter.
|
||||
* Uses relative path which will be resolved against the baseURL configured in playwright.config.ts
|
||||
* @param page - The Playwright page object
|
||||
* @param pageEnum - The page enum value to navigate to
|
||||
* Waits for the sidebar to be visible before returning.
|
||||
*/
|
||||
export async function navigateToPage(page: PlaywrightPage, pageEnum: Page): Promise<void> {
|
||||
await page.goto(`/ui?page=${pageEnum}`);
|
||||
await page.waitForLoadState("networkidle");
|
||||
// Dismiss the "Quick feedback" popup if it appears
|
||||
await dismissFeedbackPopup(page);
|
||||
}
|
||||
|
||||
/**
|
||||
* Dismiss the "Quick feedback" popup that may appear on any page.
|
||||
*/
|
||||
export async function dismissFeedbackPopup(page: PlaywrightPage): Promise<void> {
|
||||
const dismissButton = page.getByText("Don't ask me again");
|
||||
if (await dismissButton.isVisible({ timeout: 1_500 }).catch(() => false)) {
|
||||
await dismissButton.click();
|
||||
// Wait for the popup to disappear
|
||||
await expect(dismissButton).not.toBeVisible({ timeout: 2_000 }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
180
ui/litellm-dashboard/e2e_tests/run_e2e.sh
Executable file
180
ui/litellm-dashboard/e2e_tests/run_e2e.sh
Executable file
@ -0,0 +1,180 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# ================================================================
|
||||
# UI E2E Test Runner (Consolidated)
|
||||
# Starts postgres, seeds DB, starts mock + proxy, runs Playwright.
|
||||
# All tests target the proxy on port 4000 (which serves both API
|
||||
# and UI from the built Next.js static export).
|
||||
#
|
||||
# Usage:
|
||||
# ./run_e2e.sh # Run once
|
||||
# ./run_e2e.sh --repeat-each=5 # Run each test 5 times
|
||||
# ./run_e2e.sh --headed # Run with browser visible
|
||||
#
|
||||
# In CI (CI=true), expects:
|
||||
# - PostgreSQL already running on 127.0.0.1:5432
|
||||
# - DATABASE_URL already set
|
||||
# - Python/Poetry already installed
|
||||
# - Node.js/npx already available
|
||||
# ================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DASHBOARD_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
IS_CI="${CI:-false}"
|
||||
CONTAINER_NAME="litellm-e2e-postgres-$$"
|
||||
MOCK_PID=""
|
||||
PROXY_PID=""
|
||||
|
||||
# --- Ensure common tool paths are available (local dev only) ---
|
||||
if [ "$IS_CI" = "false" ]; then
|
||||
for p in /usr/local/bin /opt/homebrew/bin "$HOME/.local/bin" /opt/homebrew/opt/postgresql@14/bin /opt/homebrew/opt/libpq/bin; do
|
||||
[ -d "$p" ] && export PATH="$p:$PATH"
|
||||
done
|
||||
[ -s "$HOME/.nvm/nvm.sh" ] && source "$HOME/.nvm/nvm.sh"
|
||||
fi
|
||||
|
||||
# --- Cleanup on exit ---
|
||||
cleanup() {
|
||||
echo "Cleaning up..."
|
||||
[ -n "$MOCK_PID" ] && kill "$MOCK_PID" 2>/dev/null || true
|
||||
[ -n "$PROXY_PID" ] && kill "$PROXY_PID" 2>/dev/null || true
|
||||
if [ "$IS_CI" = "false" ]; then
|
||||
docker stop "$CONTAINER_NAME" 2>/dev/null || true
|
||||
fi
|
||||
echo "Done."
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# --- Pre-flight checks ---
|
||||
for cmd in python3 npx poetry; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; }
|
||||
done
|
||||
|
||||
# --- Database setup ---
|
||||
if [ "$IS_CI" = "false" ]; then
|
||||
for cmd in docker psql; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; }
|
||||
done
|
||||
for port in 4000 5432 8090; do
|
||||
if lsof -ti ":$port" >/dev/null 2>&1; then
|
||||
echo "Error: port $port is in use"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
export POSTGRES_USER="e2euser"
|
||||
export POSTGRES_PASSWORD="$(openssl rand -hex 32)"
|
||||
export POSTGRES_DB="litellm_e2e"
|
||||
export DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:5432/${POSTGRES_DB}"
|
||||
|
||||
echo "=== Starting PostgreSQL ==="
|
||||
docker run -d --rm --name "$CONTAINER_NAME" \
|
||||
-e POSTGRES_USER -e POSTGRES_PASSWORD -e POSTGRES_DB \
|
||||
-p 127.0.0.1:5432:5432 \
|
||||
postgres:16
|
||||
|
||||
echo "Waiting for PostgreSQL..."
|
||||
for i in $(seq 1 30); do
|
||||
if PGPASSWORD="$POSTGRES_PASSWORD" pg_isready -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
else
|
||||
echo "=== Using CI PostgreSQL service ==="
|
||||
: "${DATABASE_URL:?DATABASE_URL must be set in CI}"
|
||||
fi
|
||||
|
||||
# --- Credentials ---
|
||||
export LITELLM_MASTER_KEY="sk-1234"
|
||||
export MOCK_LLM_URL="http://127.0.0.1:8090/v1"
|
||||
export DISABLE_SCHEMA_UPDATE="true"
|
||||
# Ensure the proxy serves UI at /ui (not behind a subpath)
|
||||
export SERVER_ROOT_PATH=""
|
||||
# Prevent logout from redirecting to an external URL
|
||||
export PROXY_LOGOUT_URL=""
|
||||
|
||||
# --- Rebuild UI from source ---
|
||||
echo "=== Building UI from source ==="
|
||||
cd "$DASHBOARD_DIR"
|
||||
npm install --silent 2>/dev/null || true
|
||||
npm run build
|
||||
# Copy the fresh build to the proxy's static UI directory
|
||||
cp -r "$DASHBOARD_DIR/out/" "$REPO_ROOT/litellm/proxy/_experimental/out/"
|
||||
echo "UI build copied to proxy static directory"
|
||||
|
||||
# --- Python environment ---
|
||||
echo "=== Setting up Python environment ==="
|
||||
cd "$REPO_ROOT"
|
||||
if ! poetry run python3 -c "import prisma" 2>/dev/null; then
|
||||
echo "Installing Python dependencies (first run)..."
|
||||
poetry install --with dev,proxy-dev --extras "proxy" --quiet
|
||||
poetry run pip install nodejs-wheel-binaries 2>/dev/null || true
|
||||
poetry run prisma generate --schema litellm/proxy/schema.prisma
|
||||
fi
|
||||
|
||||
echo "=== Pushing Prisma schema to database ==="
|
||||
poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
|
||||
|
||||
# --- Mock LLM server ---
|
||||
echo "=== Starting mock LLM server ==="
|
||||
poetry run python3 "$SCRIPT_DIR/fixtures/mock_llm_server/server.py" &
|
||||
MOCK_PID=$!
|
||||
|
||||
for i in $(seq 1 15); do
|
||||
if curl -sf http://127.0.0.1:8090/health >/dev/null 2>&1; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# --- LiteLLM proxy ---
|
||||
echo "=== Starting LiteLLM proxy ==="
|
||||
cd "$REPO_ROOT"
|
||||
poetry run python3 -m litellm.proxy.proxy_cli \
|
||||
--config "$SCRIPT_DIR/fixtures/config.yml" \
|
||||
--port 4000 &
|
||||
PROXY_PID=$!
|
||||
|
||||
echo "Waiting for proxy..."
|
||||
PROXY_READY=0
|
||||
for i in $(seq 1 180); do
|
||||
if ! kill -0 "$PROXY_PID" 2>/dev/null; then
|
||||
echo "Error: proxy process exited unexpectedly"
|
||||
exit 1
|
||||
fi
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true)
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
PROXY_READY=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "$PROXY_READY" -ne 1 ]; then
|
||||
echo "Error: proxy did not become healthy within 180 seconds"
|
||||
exit 1
|
||||
fi
|
||||
echo "Proxy is ready."
|
||||
|
||||
# --- Seed database ---
|
||||
echo "=== Seeding database ==="
|
||||
DB_USER=$(echo "$DATABASE_URL" | sed -n 's|.*://\([^:]*\):.*|\1|p')
|
||||
DB_PASS=$(echo "$DATABASE_URL" | sed -n 's|.*://[^:]*:\([^@]*\)@.*|\1|p')
|
||||
DB_HOST=$(echo "$DATABASE_URL" | sed -n 's|.*@\([^:]*\):.*|\1|p')
|
||||
DB_PORT=$(echo "$DATABASE_URL" | sed -n 's|.*:\([0-9]*\)/.*|\1|p')
|
||||
DB_NAME=$(echo "$DATABASE_URL" | sed -n 's|.*/\([^?]*\).*|\1|p')
|
||||
|
||||
PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
|
||||
-f "$SCRIPT_DIR/fixtures/seed.sql"
|
||||
|
||||
# --- Playwright ---
|
||||
echo "=== Installing Playwright dependencies ==="
|
||||
cd "$DASHBOARD_DIR"
|
||||
npm install --silent 2>/dev/null || true
|
||||
npx playwright install chromium --with-deps 2>/dev/null || npx playwright install chromium
|
||||
|
||||
echo "=== Running Playwright tests ==="
|
||||
npx playwright test --config e2e_tests/playwright.config.ts "$@"
|
||||
EXIT_CODE=$?
|
||||
|
||||
exit $EXIT_CODE
|
||||
124
ui/litellm-dashboard/e2e_tests/tests/proxy-admin/keys.spec.ts
Normal file
124
ui/litellm-dashboard/e2e_tests/tests/proxy-admin/keys.spec.ts
Normal file
@ -0,0 +1,124 @@
|
||||
import { test, expect } from "@playwright/test";
|
||||
import {
|
||||
ADMIN_STORAGE_PATH,
|
||||
E2E_DELETE_KEY_ALIAS,
|
||||
E2E_REGENERATE_KEY_ALIAS,
|
||||
E2E_UPDATE_LIMITS_KEY_ALIAS,
|
||||
E2E_INTERNAL_USER_KEY_ALIAS,
|
||||
E2E_TEAM_CRUD_ALIAS,
|
||||
} from "../../constants";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
|
||||
|
||||
test.describe("Proxy Admin - Keys", () => {
|
||||
test.use({ storageState: ADMIN_STORAGE_PATH });
|
||||
|
||||
test("Create a key in a team", async ({ page }) => {
|
||||
await navigateToPage(page, Page.ApiKeys);
|
||||
await dismissFeedbackPopup(page);
|
||||
|
||||
// Click "+ Create New Key" button
|
||||
await page.getByRole("button", { name: /Create New Key/i }).click();
|
||||
|
||||
// Wait for the key creation modal
|
||||
await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
// Fill key name (has data-testid="base-input" in the built UI)
|
||||
const keyName = `e2e-admin-key-${Date.now()}`;
|
||||
await page.getByTestId("base-input").fill(keyName);
|
||||
|
||||
// Select team — the team dropdown has placeholder "Search or select a team"
|
||||
const teamSelect = page.locator(".ant-select", { hasText: "Search or select a team" });
|
||||
await teamSelect.click();
|
||||
await page.keyboard.type(E2E_TEAM_CRUD_ALIAS);
|
||||
await page.locator(".ant-select-dropdown:visible").getByText(E2E_TEAM_CRUD_ALIAS).first().click();
|
||||
|
||||
// Select models
|
||||
await page.locator(".ant-select-selection-overflow").click();
|
||||
await page.locator(".ant-select-dropdown:visible").getByText("All Team Models").click();
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
// Submit
|
||||
await page.getByRole("button", { name: "Create Key", exact: true }).click();
|
||||
|
||||
// Success shows "Save your Key" in a second dialog
|
||||
await expect(page.getByText("Save your Key")).toBeVisible({ timeout: 10_000 });
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
// Verify the new key appears in the table
|
||||
await expect(page.getByText(keyName)).toBeVisible({ timeout: 10_000 });
|
||||
});
|
||||
|
||||
test("Regenerate key", async ({ page }) => {
|
||||
await navigateToPage(page, Page.ApiKeys);
|
||||
await dismissFeedbackPopup(page);
|
||||
|
||||
// Key IDs are rendered as buttons in the table
|
||||
const keyRow = page.locator("tr", { hasText: E2E_REGENERATE_KEY_ALIAS });
|
||||
await expect(keyRow).toBeVisible({ timeout: 10_000 });
|
||||
await keyRow.locator("button").first().click();
|
||||
|
||||
await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
await page.getByRole("button", { name: "Regenerate Key" }).click();
|
||||
await page.getByRole("button", { name: "Regenerate", exact: true }).click();
|
||||
|
||||
// Success shows "Copy Virtual Key" button in the regenerated key dialog
|
||||
await expect(page.getByText("Copy Virtual Key")).toBeVisible({ timeout: 10_000 });
|
||||
});
|
||||
|
||||
test("Update key TPM and RPM limits", async ({ page }) => {
|
||||
await navigateToPage(page, Page.ApiKeys);
|
||||
await dismissFeedbackPopup(page);
|
||||
|
||||
const keyRow = page.locator("tr", { hasText: E2E_UPDATE_LIMITS_KEY_ALIAS });
|
||||
await expect(keyRow).toBeVisible({ timeout: 10_000 });
|
||||
await keyRow.locator("button").first().click();
|
||||
|
||||
await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
await page.getByRole("tab", { name: "Settings" }).click();
|
||||
await page.getByRole("button", { name: "Edit Settings" }).click();
|
||||
|
||||
await page.getByRole("spinbutton", { name: "TPM Limit" }).fill("123");
|
||||
await page.getByRole("spinbutton", { name: "RPM Limit" }).fill("456");
|
||||
await page.getByRole("button", { name: "Save Changes" }).click();
|
||||
|
||||
await expect(
|
||||
page.getByRole("paragraph").filter({ hasText: "TPM: 123" })
|
||||
).toBeVisible({ timeout: 10_000 });
|
||||
await expect(
|
||||
page.getByRole("paragraph").filter({ hasText: "RPM: 456" })
|
||||
).toBeVisible({ timeout: 10_000 });
|
||||
});
|
||||
|
||||
test("Delete key", async ({ page }) => {
|
||||
await navigateToPage(page, Page.ApiKeys);
|
||||
await dismissFeedbackPopup(page);
|
||||
|
||||
const keyRow = page.locator("tr", { hasText: E2E_DELETE_KEY_ALIAS });
|
||||
await expect(keyRow).toBeVisible({ timeout: 10_000 });
|
||||
await keyRow.locator("button").first().click();
|
||||
|
||||
await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
await page.getByRole("button", { name: "Delete Key" }).click();
|
||||
|
||||
const modal = page.locator(".ant-modal:visible");
|
||||
await expect(modal).toBeVisible({ timeout: 5_000 });
|
||||
await modal.locator("input").fill(E2E_DELETE_KEY_ALIAS);
|
||||
|
||||
const deleteButton = modal.getByRole("button", { name: "Delete", exact: true });
|
||||
await expect(deleteButton).toBeEnabled();
|
||||
await deleteButton.click();
|
||||
|
||||
await expect(page.getByText(/Key deleted/i).first()).toBeVisible({ timeout: 10_000 });
|
||||
});
|
||||
|
||||
test("See internal user keys in team", async ({ page }) => {
|
||||
await navigateToPage(page, Page.ApiKeys);
|
||||
await dismissFeedbackPopup(page);
|
||||
|
||||
await expect(page.getByText(E2E_INTERNAL_USER_KEY_ALIAS)).toBeVisible({ timeout: 10_000 });
|
||||
});
|
||||
});
|
||||
136
ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts
Normal file
136
ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts
Normal file
@ -0,0 +1,136 @@
|
||||
import { test, expect } from "@playwright/test";
|
||||
import {
|
||||
ADMIN_STORAGE_PATH,
|
||||
E2E_TEAM_CRUD_ID,
|
||||
E2E_TEAM_DELETE_ALIAS,
|
||||
E2E_TEAM_NO_ADMIN_ID,
|
||||
E2E_TEAM_ORG_ID,
|
||||
} from "../../constants";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
|
||||
|
||||
/**
|
||||
* Click on a team ID in the table. Team IDs are rendered differently depending
|
||||
* on the component version — try button first (Tremor Button), fall back to
|
||||
* clickable span (OldTeams Typography.Text).
|
||||
*/
|
||||
async function clickTeamId(page: import("@playwright/test").Page, teamId: string) {
|
||||
const idPrefix = teamId.slice(0, 7);
|
||||
// The team ID is either a Button or a clickable span containing the first 7 chars
|
||||
const cell = page.locator("td").filter({ hasText: teamId }).first();
|
||||
await expect(cell).toBeVisible({ timeout: 10_000 });
|
||||
await cell.click();
|
||||
await expect(page.getByText("Back to Teams")).toBeVisible({ timeout: 10_000 });
|
||||
}
|
||||
|
||||
test.describe("Proxy Admin - Teams", () => {
|
||||
test.use({ storageState: ADMIN_STORAGE_PATH });
|
||||
|
||||
test("Create a team", async ({ page }) => {
|
||||
await navigateToPage(page, Page.Teams);
|
||||
await dismissFeedbackPopup(page);
|
||||
|
||||
const uniqueAlias = `e2e-created-team-${Date.now()}`;
|
||||
|
||||
// Click the Create Team button — accessible name includes "Create Team"
|
||||
await page.getByRole("button", { name: /Create Team/i }).first().click();
|
||||
|
||||
// Wait for the Create Team modal
|
||||
const dialog = page.locator(".ant-modal:visible");
|
||||
await expect(dialog).toBeVisible({ timeout: 5_000 });
|
||||
|
||||
// Fill Team Name — the input has id="team_alias"
|
||||
await dialog.locator("#team_alias").fill(uniqueAlias);
|
||||
|
||||
// Select models — the models multi-select is inside the modal
|
||||
// Click to open dropdown, select "All Proxy Models"
|
||||
await dialog.locator(".ant-select-selection-overflow").first().click();
|
||||
await page.locator(".ant-select-dropdown:visible").getByText("All Proxy Models").click();
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
// Submit — click the submit button inside the dialog (not the header button)
|
||||
await dialog.locator("button[type='submit']").click();
|
||||
|
||||
// Verify success notification
|
||||
await expect(page.getByText("Team created").first()).toBeVisible({ timeout: 10_000 });
|
||||
});
|
||||
|
||||
test("Invite a user to a team", async ({ page }) => {
|
||||
await navigateToPage(page, Page.Teams);
|
||||
await dismissFeedbackPopup(page);
|
||||
|
||||
await clickTeamId(page, E2E_TEAM_CRUD_ID);
|
||||
|
||||
await page.getByRole("tab", { name: "Members" }).click();
|
||||
await page.getByRole("button", { name: /Add Member/i }).click();
|
||||
|
||||
// Wait for Add Team Member modal
|
||||
const modal = page.locator(".ant-modal:visible");
|
||||
await expect(modal).toBeVisible({ timeout: 5_000 });
|
||||
|
||||
// The email field is a Select — type to search, then select from dropdown
|
||||
await modal.locator(".ant-select").first().click();
|
||||
await page.keyboard.type("invitable@test.local");
|
||||
|
||||
// Wait for the option to appear, then select via keyboard (avoids viewport issues)
|
||||
const emailOption = page.getByRole("option", { name: "invitable@test.local" }).first();
|
||||
await expect(emailOption).toBeAttached({ timeout: 10_000 });
|
||||
// Use keyboard to select the highlighted option
|
||||
await page.keyboard.press("Enter");
|
||||
|
||||
// Submit
|
||||
await modal.getByRole("button", { name: /Add Member/i }).click();
|
||||
|
||||
await expect(page.getByText(/member.*added|success/i).first()).toBeVisible({ timeout: 10_000 });
|
||||
});
|
||||
|
||||
test("Edit team member for team proxy admin does not belong to", async ({ page }) => {
|
||||
await navigateToPage(page, Page.Teams);
|
||||
await dismissFeedbackPopup(page);
|
||||
|
||||
await clickTeamId(page, E2E_TEAM_NO_ADMIN_ID);
|
||||
|
||||
await page.getByRole("tab", { name: "Members" }).click();
|
||||
|
||||
await page.getByTestId("edit-member").first().click();
|
||||
|
||||
const modal = page.locator(".ant-modal:visible");
|
||||
await expect(modal).toBeVisible({ timeout: 5_000 });
|
||||
await modal.getByRole("button", { name: /Save Changes/i }).click();
|
||||
|
||||
await expect(page.getByText(/updated|success/i).first()).toBeVisible({ timeout: 10_000 });
|
||||
});
|
||||
|
||||
test("Delete a team", async ({ page }) => {
|
||||
await navigateToPage(page, Page.Teams);
|
||||
await dismissFeedbackPopup(page);
|
||||
|
||||
const teamRow = page.locator("tr", { hasText: E2E_TEAM_DELETE_ALIAS }).first();
|
||||
await expect(teamRow).toBeVisible({ timeout: 10_000 });
|
||||
await teamRow.locator("svg, img").last().click();
|
||||
|
||||
const modal = page.locator(".ant-modal:visible");
|
||||
await expect(modal).toBeVisible({ timeout: 5_000 });
|
||||
await modal.locator("input").fill(E2E_TEAM_DELETE_ALIAS);
|
||||
await modal.getByRole("button", { name: /Force Delete|Delete/i }).click();
|
||||
|
||||
await expect(teamRow).not.toBeVisible({ timeout: 10_000 });
|
||||
});
|
||||
|
||||
test("Team in org - edit team member", async ({ page }) => {
|
||||
await navigateToPage(page, Page.Teams);
|
||||
await dismissFeedbackPopup(page);
|
||||
|
||||
await clickTeamId(page, E2E_TEAM_ORG_ID);
|
||||
|
||||
await page.getByRole("tab", { name: "Members" }).click();
|
||||
|
||||
await page.getByTestId("edit-member").first().click();
|
||||
|
||||
const modal = page.locator(".ant-modal:visible");
|
||||
await expect(modal).toBeVisible({ timeout: 5_000 });
|
||||
await modal.getByRole("button", { name: /Save Changes/i }).click();
|
||||
|
||||
await expect(page.getByText(/updated|success/i).first()).toBeVisible({ timeout: 10_000 });
|
||||
});
|
||||
});
|
||||
@ -80,6 +80,7 @@ const TeamsTable = ({
|
||||
size="xs"
|
||||
variant="light"
|
||||
className="font-mono text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs font-normal px-2 py-0.5 text-left overflow-hidden truncate max-w-[200px]"
|
||||
data-testid="team-id-cell"
|
||||
onClick={() => {
|
||||
// Add click handler
|
||||
setSelectedTeamId(team.team_id);
|
||||
|
||||
@ -312,7 +312,7 @@ const CreateTeamModal = ({
|
||||
},
|
||||
]}
|
||||
>
|
||||
<TextInput placeholder="" />
|
||||
<TextInput placeholder="" data-testid="team-name-input" />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label={
|
||||
@ -379,7 +379,7 @@ const CreateTeamModal = ({
|
||||
}
|
||||
name="models"
|
||||
>
|
||||
<Select2 mode="multiple" placeholder="Select models" style={{ width: "100%" }}>
|
||||
<Select2 mode="multiple" placeholder="Select models" style={{ width: "100%" }} data-testid="team-models-select">
|
||||
<Select2.Option key="all-proxy-models" value="all-proxy-models">
|
||||
All Proxy Models
|
||||
</Select2.Option>
|
||||
@ -716,7 +716,7 @@ const CreateTeamModal = ({
|
||||
</Accordion>
|
||||
</>
|
||||
<div style={{ textAlign: "right", marginTop: "10px" }}>
|
||||
<Button2 htmlType="submit">Create Team</Button2>
|
||||
<Button2 htmlType="submit" data-testid="create-team-submit">Create Team</Button2>
|
||||
</div>
|
||||
</Form>
|
||||
</Modal>
|
||||
|
||||
@ -695,6 +695,7 @@ const Teams: React.FC<TeamProps> = ({
|
||||
className="text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs cursor-pointer"
|
||||
style={{ fontSize: 14, padding: "1px 8px" }}
|
||||
onClick={() => setSelectedTeamId(record.team_id)}
|
||||
data-testid="team-id-cell"
|
||||
>
|
||||
{id}
|
||||
</Text>
|
||||
@ -898,6 +899,7 @@ const Teams: React.FC<TeamProps> = ({
|
||||
icon={<PlusOutlined />}
|
||||
onClick={() => setIsTeamModalVisible(true)}
|
||||
style={{ marginTop: 16 }}
|
||||
data-testid="create-team-button"
|
||||
>
|
||||
Create Team
|
||||
</Button>
|
||||
@ -1041,7 +1043,7 @@ const Teams: React.FC<TeamProps> = ({
|
||||
</Text>
|
||||
</Space>
|
||||
{canCreateOrManageTeams(userRole, userID, organizations) && (
|
||||
<Button type="primary" icon={<PlusOutlined />} onClick={() => setIsTeamModalVisible(true)}>
|
||||
<Button type="primary" icon={<PlusOutlined />} onClick={() => setIsTeamModalVisible(true)} data-testid="create-team-button">
|
||||
Create Team
|
||||
</Button>
|
||||
)}
|
||||
@ -1078,7 +1080,7 @@ const Teams: React.FC<TeamProps> = ({
|
||||
},
|
||||
]}
|
||||
>
|
||||
<TextInput placeholder="" />
|
||||
<TextInput placeholder="" data-testid="team-name-input" />
|
||||
</Form.Item>
|
||||
{(() => {
|
||||
const adminOrgs = getAdminOrganizations(userRole, userID, organizations);
|
||||
@ -1567,7 +1569,7 @@ const Teams: React.FC<TeamProps> = ({
|
||||
</Accordion>
|
||||
</>
|
||||
<div style={{ textAlign: "right", marginTop: "10px" }}>
|
||||
<Button htmlType="submit">Create Team</Button>
|
||||
<Button htmlType="submit" data-testid="create-team-submit">Create Team</Button>
|
||||
</div>
|
||||
</Form>
|
||||
</Modal>
|
||||
|
||||
@ -96,6 +96,7 @@ const TeamDropdown: React.FC<TeamDropdownProps> = ({
|
||||
onPopupScroll={handlePopupScroll}
|
||||
loading={isLoading}
|
||||
notFoundContent={isLoading ? <LoadingOutlined spin /> : "No teams found"}
|
||||
data-testid="team-dropdown"
|
||||
popupRender={(menu) => (
|
||||
<>
|
||||
{menu}
|
||||
|
||||
@ -150,6 +150,7 @@ const UserSearchModal: React.FC<UserSearchModalProps> = ({
|
||||
options={selectedField === "user_email" ? userOptions : []}
|
||||
loading={loading}
|
||||
allowClear
|
||||
data-testid="member-email-search"
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
|
||||
@ -666,7 +666,7 @@ const CreateKey: React.FC<CreateKeyProps> = ({ team, teams, data, addKey, autoOp
|
||||
return (
|
||||
<div>
|
||||
{userRole && rolesWithWriteAccess.includes(userRole) && (
|
||||
<Button className="mx-auto" onClick={() => setIsModalVisible(true)}>
|
||||
<Button className="mx-auto" onClick={() => setIsModalVisible(true)} data-testid="create-key-button">
|
||||
+ Create New Key
|
||||
</Button>
|
||||
)}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user