62dca9e977
* fix(ci): flag codecov uploads and enable carryforward Coverage uploads from GHA and CircleCI were unflagged. Commits that receive the push-triggered workflows more than once (re-runs, or branches cut at the same SHA) accumulated many overlapping flagless sessions, and Codecov's per-commit merge dropped the largest, ubiquitously-imported files (router.py, proxy_server.py, main.py, utils.py, cost_calculator.py) from the report even though the uploaded XMLs contained them. - codecov.yaml: flag_management.default_rules.carryforward: true - GHA reusable bases: tag each upload with its workflow/shard name - CircleCI: tag the combined upload "circleci"; also combine the agent / google_generate_content_endpoint / litellm_utils datafiles that were produced and required but missing from the combine list * fix(ci): close coverage gaps in proxy-legacy, router-unit, auth-ui, caching-redis - test-unit-proxy-legacy: route through _test-unit-base so the full proxy_unit_tests suite (incl. comprehensive test_proxy_server*.py) is measured and uploaded with per-group flags (was plain pytest, no --cov) - _test-unit-services-base: declare the enable-redis input + the six secrets test-unit-caching-redis passes; that workflow had a workflow_call signature mismatch and startup_failed on every push (never ran). Changes are additive/optional - proxy-db and security callers unchanged - circleci: add --cov + persist + combine + upload-coverage requires for litellm_router_unit_testing (tests/router_unit_tests) and auth_ui_unit_tests (tests/proxy_admin_ui_tests); neither was covered anywhere. Redundant -k subset jobs left as-is (local_testing covers them) * fix(ci): remove dead GHA Redis workflow; keep Redis on CircleCI only CircleCI redis_caching_unit_tests already runs the exact same files (tests/local_testing/test_dual_cache.py, test_redis_batch_optimizations.py, test_router_utils.py) with --cov, and that datafile is already combined and uploaded. The GHA test-unit-caching-redis workflow was redundant and had never run (workflow_call signature mismatch -> startup_failure on every push). - Delete .github/workflows/test-unit-caching-redis.yml - Revert _test-unit-services-base.yml to the flag-fix state (drop the enable-redis input / secrets / env wiring added only to prop up the GHA Redis workflow); the verified per-upload flags line is kept - The only single-star "litellm_*" branch glob lived in the deleted file; no other single-star globs exist, so none remain to widen * fix(ci): keep proxy-legacy as a standalone job to preserve required check names Routing proxy-legacy through the reusable workflow renamed each check from the bare matrix name (e.g. "proxy-response-and-misc") to "proxy-response-and-misc / Run tests". Those bare names are required status checks in branch protection, so the old contexts never reported and PRs sat "Expected — Waiting for status to be reported" indefinitely. Restore the original standalone matrix job (job name == matrix name, so the required contexts report again) and add coverage in place: --cov on pytest plus an OIDC Codecov upload flagged proxy-legacy-<group>. Net effect of the gap-#2 fix is preserved (flagged coverage for tests/proxy_unit_tests/**) without changing any check name. * revert(ci): drop all proxy-legacy changes from this PR tests/proxy_unit_tests/** is already fully covered by test-unit-proxy-db (its shard-coverage guard fails CI if any file in that dir is unassigned), which this PR already flags + carryforwards. Adding --cov and id-token:write to the legacy pull_request job was redundant and put OIDC on a job that runs untrusted PR code. Restore the file to the base version verbatim so this PR no longer touches proxy-legacy at all (also restores its original required check names). Retiring proxy-legacy in favor of proxy-db on pull_request is a separate effort that needs a branch-protection change.
2715 lines
98 KiB
YAML
2715 lines
98 KiB
YAML
version: 2.1
|
|
orbs:
|
|
codecov: codecov/codecov@4.0.1
|
|
node: circleci/node@5.1.0 # Add this line to declare the node orb
|
|
win: circleci/windows@5.0 # Add Windows orb
|
|
|
|
commands:
|
|
setup_google_dns:
|
|
steps:
|
|
- run:
|
|
name: "Configure Google DNS"
|
|
command: |
|
|
# Backup original resolv.conf
|
|
sudo cp /etc/resolv.conf /etc/resolv.conf.backup
|
|
# Add both local and Google DNS servers
|
|
echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf
|
|
echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf
|
|
echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf
|
|
wait_for_service:
|
|
description: "Poll a TCP or HTTP endpoint until it responds (replaces dockerize -wait)"
|
|
parameters:
|
|
url:
|
|
type: string
|
|
timeout:
|
|
type: string
|
|
default: "60"
|
|
steps:
|
|
- run:
|
|
name: "Wait for << parameters.url >>"
|
|
command: |
|
|
TIMEOUT=<< parameters.timeout >>
|
|
URL="<< parameters.url >>"
|
|
ELAPSED=0
|
|
echo "Waiting up to ${TIMEOUT}s for ${URL} ..."
|
|
if echo "$URL" | grep -q '^tcp://'; then
|
|
HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1)
|
|
PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2)
|
|
while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do
|
|
sleep 2; ELAPSED=$((ELAPSED+2))
|
|
if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
|
|
done
|
|
else
|
|
while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do
|
|
sleep 2; ELAPSED=$((ELAPSED+2))
|
|
if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
|
|
done
|
|
fi
|
|
echo "Service ready after ${ELAPSED}s"
|
|
install_helm:
|
|
steps:
|
|
- run:
|
|
name: Install Helm v3.17.3
|
|
command: |
|
|
curl -sSLf -o /tmp/helm.tar.gz \
|
|
https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz
|
|
echo "ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd /tmp/helm.tar.gz" | sha256sum -c -
|
|
sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/helm.tar.gz linux-amd64/helm
|
|
rm -f /tmp/helm.tar.gz
|
|
install_kind:
|
|
steps:
|
|
- run:
|
|
name: Install Kind v0.20.0
|
|
command: |
|
|
curl -sSLf -o /tmp/kind \
|
|
https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
|
|
echo "513a7213d6d3332dd9ef27c24dab35e5ef10a04fa27274fe1c14d8a246493ded /tmp/kind" | sha256sum -c -
|
|
chmod +x /tmp/kind
|
|
sudo mv /tmp/kind /usr/local/bin/kind
|
|
install_uv:
|
|
description: "Install pinned uv (0.10.9) with checksum verification. Adds ~/.local/bin to PATH."
|
|
steps:
|
|
- run:
|
|
name: Install uv (pinned 0.10.9)
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
start_postgres:
|
|
description: "Start a postgres-db container on port 5432 and wait until it accepts connections."
|
|
parameters:
|
|
db_name:
|
|
type: string
|
|
default: circle_test
|
|
steps:
|
|
- run:
|
|
name: Start PostgreSQL
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=<< parameters.db_name >> \
|
|
-p 5432:5432 \
|
|
postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
|
|
- wait_for_service:
|
|
url: tcp://localhost:5432
|
|
timeout: "60"
|
|
start_redis:
|
|
description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys."
|
|
steps:
|
|
- run:
|
|
name: Start Redis
|
|
command: |
|
|
docker run -d \
|
|
--name redis-cache \
|
|
-p 6379:6379 \
|
|
redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
|
|
- wait_for_service:
|
|
url: tcp://localhost:6379
|
|
timeout: "60"
|
|
setup_litellm_enterprise_pip:
|
|
steps:
|
|
- run:
|
|
name: "Install local version of litellm-enterprise"
|
|
command: |
|
|
# litellm-enterprise is a uv workspace member and is already installed
|
|
# by the main `uv sync --all-groups --all-extras`. Do NOT run
|
|
# `uv sync --package litellm-enterprise` here — that overwrites the
|
|
# shared .venv and strips out dev/test deps (pytest, prisma, etc.).
|
|
uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
|
|
setup_litellm_test_deps:
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
|
|
jobs:
|
|
# Add Windows testing job
|
|
using_litellm_on_windows:
|
|
executor:
|
|
name: win/default
|
|
shell: powershell.exe
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install Python
|
|
command: |
|
|
choco install python --version=3.11.0 -y --no-progress --force
|
|
refreshenv
|
|
python --version
|
|
environment:
|
|
CHOCOLATEY_CONFIRM_ALL: "true"
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
$installer = Join-Path $env:TEMP "uv-install.ps1"
|
|
Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer
|
|
$expected = "d43ffff8d28e7d1e7d1831a212465f12b24a43c7f87f386e95e2a5915aee5d7d"
|
|
$actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower()
|
|
if ($actual -ne $expected) {
|
|
throw "uv installer hash mismatch: expected $expected got $actual"
|
|
}
|
|
& $installer
|
|
Remove-Item $installer
|
|
$uvBin = Join-Path $HOME ".local\bin"
|
|
$env:Path = "$uvBin;$env:Path"
|
|
if (!(Test-Path $PROFILE)) {
|
|
New-Item -ItemType File -Force -Path $PROFILE | Out-Null
|
|
}
|
|
if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) {
|
|
Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`""
|
|
}
|
|
uv sync --frozen --group dev --python (Get-Command python).Source
|
|
- run:
|
|
name: Run Windows-specific test
|
|
command: |
|
|
uv run --no-sync python -m pytest tests/windows_tests/test_litellm_on_windows.py -v
|
|
|
|
local_testing_part1:
|
|
docker:
|
|
- &python312_image
|
|
image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
parallelism: 4
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Run prisma ./docker/entrypoint.sh
|
|
command: |
|
|
set +e
|
|
chmod +x docker/entrypoint.sh
|
|
./docker/entrypoint.sh
|
|
set -e
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests (Part 1 - A-M)
|
|
command: |
|
|
mkdir test-results
|
|
|
|
# Discover test files (A-M)
|
|
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py")
|
|
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--split-by=timings \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv \
|
|
--cov=./litellm \
|
|
--cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=20 \
|
|
-k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
|
|
-n 4 \
|
|
--timeout=300 \
|
|
--timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml local_testing_part1_coverage.xml
|
|
mv .coverage local_testing_part1_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- local_testing_part1_coverage.xml
|
|
- local_testing_part1_coverage
|
|
local_testing_part2:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
parallelism: 4
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Run prisma ./docker/entrypoint.sh
|
|
command: |
|
|
set +e
|
|
chmod +x docker/entrypoint.sh
|
|
./docker/entrypoint.sh
|
|
set -e
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests (Part 2 - N-Z)
|
|
command: |
|
|
mkdir test-results
|
|
|
|
# Discover test files (N-Z)
|
|
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py")
|
|
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--split-by=timings \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv \
|
|
--cov=./litellm \
|
|
--cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=20 \
|
|
-k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
|
|
-n 4 \
|
|
--timeout=300 \
|
|
--timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml local_testing_part2_coverage.xml
|
|
mv .coverage local_testing_part2_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- local_testing_part2_coverage.xml
|
|
- local_testing_part2_coverage
|
|
langfuse_logging_unit_tests:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: medium
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Run prisma ./docker/entrypoint.sh
|
|
command: |
|
|
set +e
|
|
chmod +x docker/entrypoint.sh
|
|
./docker/entrypoint.sh
|
|
set -e
|
|
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-k \"langfuse\""
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
auth_ui_unit_tests:
|
|
docker:
|
|
- *python312_image
|
|
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
|
|
environment:
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_DB: litellm_test
|
|
working_directory: ~/project
|
|
environment:
|
|
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- wait_for_service:
|
|
url: tcp://localhost:5432
|
|
timeout: "60"
|
|
- run:
|
|
name: Seed DB schema via prisma db push
|
|
command: |
|
|
set +e
|
|
uv run --no-sync litellm --skip_server_startup --use_prisma_db_push
|
|
set -e
|
|
- run:
|
|
name: Generate Prisma Client
|
|
command: uv run --no-sync python -m prisma generate
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--cov=./litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 2"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml auth_ui_unit_tests_coverage.xml
|
|
mv .coverage auth_ui_unit_tests_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- auth_ui_unit_tests_coverage.xml
|
|
- auth_ui_unit_tests_coverage
|
|
|
|
litellm_router_testing: # Runs all tests with the "router" keyword
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
parallelism: 4
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
# Run pytest and generate JUnit XML report
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
|
|
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--split-by=timings \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v \
|
|
-k 'router' \
|
|
-n 4 \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
--timeout=300 --timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
litellm_router_unit_testing: # Runs all tests with the "router" keyword
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
# Run pytest and generate JUnit XML report
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--cov=./litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 4"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml router_unit_tests_coverage.xml
|
|
mv .coverage router_unit_tests_coverage
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- router_unit_tests_coverage.xml
|
|
- router_unit_tests_coverage
|
|
litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: medium
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-k \"assistants\""
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
llm_translation_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: xlarge
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
# Add --timeout to kill hanging tests after 120s (2 min)
|
|
# Add --durations=20 to show 20 slowest tests for debugging
|
|
# Subdirectories with dedicated jobs (maintain this list as new jobs are added)
|
|
mkdir -p test-results
|
|
# Glob excludes the realtime/ subdirectory since it has its own job
|
|
TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=20 \
|
|
-n 4 \
|
|
--timeout=120 --timeout_method=thread \
|
|
--retries 2 --retry-delay 5 \
|
|
--max-worker-restart=5"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
realtime_translation_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run realtime tests
|
|
command: |
|
|
# Add --timeout to kill hanging tests after 120s (2 min)
|
|
# Add --durations=20 to show 20 slowest tests for debugging
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv \
|
|
--cov=./litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=20 \
|
|
-n 4 \
|
|
--timeout=120 --timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml realtime_translation_coverage.xml
|
|
mv .coverage realtime_translation_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- realtime_translation_coverage.xml
|
|
- realtime_translation_coverage
|
|
agent_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x -s \
|
|
--cov=./litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml agent_coverage.xml
|
|
mv .coverage agent_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- agent_coverage.xml
|
|
- agent_coverage
|
|
guardrails_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
export LITELLM_LOG=WARNING
|
|
TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv \
|
|
--cov=./litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 2 \
|
|
--timeout=120 --timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml guardrails_coverage.xml
|
|
mv .coverage guardrails_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- guardrails_coverage.xml
|
|
- guardrails_coverage
|
|
|
|
google_generate_content_endpoint_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x -s \
|
|
--cov=./litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
--retries 3 --retry-delay 5"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml google_generate_content_endpoint_coverage.xml
|
|
mv .coverage google_generate_content_endpoint_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- google_generate_content_endpoint_coverage.xml
|
|
- google_generate_content_endpoint_coverage
|
|
|
|
llm_responses_api_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 8"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
ocr_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x \
|
|
--cov=./litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 4"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml ocr_coverage.xml
|
|
mv .coverage ocr_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- ocr_coverage.xml
|
|
- ocr_coverage
|
|
search_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x \
|
|
--cov=./litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 4"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml search_coverage.xml
|
|
mv .coverage search_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- search_coverage.xml
|
|
- search_coverage
|
|
litellm_mapped_enterprise_tests:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run enterprise tests
|
|
command: |
|
|
uv run --no-sync python -m prisma generate
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit-enterprise.xml \
|
|
--durations=10 \
|
|
-n 4"
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
batches_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x -s \
|
|
--cov=./litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 2"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml batches_coverage.xml
|
|
mv .coverage batches_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- batches_coverage.xml
|
|
- batches_coverage
|
|
litellm_utils_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x -s \
|
|
--cov=./litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 2"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml litellm_utils_coverage.xml
|
|
mv .coverage litellm_utils_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- litellm_utils_coverage.xml
|
|
- litellm_utils_coverage
|
|
|
|
pass_through_unit_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x \
|
|
--cov=./litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 4"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml pass_through_unit_tests_coverage.xml
|
|
mv .coverage pass_through_unit_tests_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- pass_through_unit_tests_coverage.xml
|
|
- pass_through_unit_tests_coverage
|
|
image_gen_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 4"
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
logging_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
export LITELLM_LOG=WARNING
|
|
TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv \
|
|
--cov=./litellm --cov-report=xml \
|
|
-n 4 \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
--timeout=120 --timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml logging_coverage.xml || true
|
|
mv .coverage logging_coverage || true
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- logging_coverage.xml
|
|
- logging_coverage
|
|
audio_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x -s \
|
|
--cov=./litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml audio_coverage.xml
|
|
mv .coverage audio_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- audio_coverage.xml
|
|
- audio_coverage
|
|
redis_caching_unit_tests:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(printf "%s\n" \
|
|
tests/local_testing/test_dual_cache.py \
|
|
tests/local_testing/test_redis_batch_optimizations.py \
|
|
tests/local_testing/test_router_utils.py)
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x -s \
|
|
--cov=./litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 -n 2 \
|
|
--reruns 2 --reruns-delay 1"
|
|
no_output_timeout: 20m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml redis_caching_coverage.xml
|
|
mv .coverage redis_caching_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- redis_caching_coverage.xml
|
|
- redis_caching_coverage
|
|
installing_litellm_on_python:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
|
|
|
|
installing_litellm_on_python_3_13:
|
|
docker:
|
|
- image: cimg/python:3.13.1@sha256:87b243ae80d154db75ce5e58af16c72c5dd4b1e23e5c7264a816e85e0c440c13
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: medium
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.13
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
|
|
|
|
installing_litellm_on_python_v2_migration_resolver:
|
|
docker:
|
|
- *python312_image
|
|
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
|
|
environment:
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_DB: litellm_test
|
|
working_directory: ~/project
|
|
environment:
|
|
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- wait_for_service:
|
|
url: tcp://localhost:5432
|
|
timeout: "60"
|
|
- run:
|
|
name: Run v2 migration resolver proxy smoke test
|
|
command: |
|
|
uv run --no-sync python -m pytest -vv \
|
|
tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver
|
|
|
|
helm_chart_testing:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker
|
|
resource_class: medium
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- setup_google_dns
|
|
- install_helm
|
|
- install_kind
|
|
|
|
# Install kubectl (pinned version with hardcoded checksum)
|
|
- run:
|
|
name: Install kubectl v1.31.4
|
|
command: |
|
|
curl -sSLf -o /tmp/kubectl \
|
|
https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl
|
|
echo "298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f /tmp/kubectl" | sha256sum -c -
|
|
chmod +x /tmp/kubectl
|
|
sudo mv /tmp/kubectl /usr/local/bin/
|
|
|
|
# Create kind cluster
|
|
- run:
|
|
name: Create Kind Cluster
|
|
command: |
|
|
kind create cluster --name litellm-test
|
|
|
|
- run:
|
|
name: Load Docker Database Image for helm tests
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
|
docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG}
|
|
|
|
- run:
|
|
name: Load Docker image into Kind
|
|
command: |
|
|
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
|
kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test
|
|
|
|
# Run helm lint
|
|
- run:
|
|
name: Run helm lint
|
|
command: |
|
|
helm lint ./deploy/charts/litellm-helm
|
|
|
|
# Run helm tests
|
|
- run:
|
|
name: Run helm tests
|
|
command: |
|
|
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
|
helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \
|
|
--set image.repository=litellm-ci \
|
|
--set image.tag=${IMAGE_TAG} \
|
|
--set image.pullPolicy=Never
|
|
# Wait for pod to be ready
|
|
echo "Waiting 30 seconds for pod to be ready..."
|
|
sleep 30
|
|
|
|
# Print pod logs before running tests
|
|
echo "Printing pod logs..."
|
|
kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}")
|
|
|
|
# Run the helm tests
|
|
helm test litellm --logs
|
|
|
|
# Cleanup
|
|
- run:
|
|
name: Cleanup
|
|
command: |
|
|
kind delete cluster --name litellm-test
|
|
when: always # This ensures cleanup runs even if previous steps fail
|
|
|
|
db_migration_disable_update_check:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: medium
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres:
|
|
db_name: litellm_test
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Seed database with real schema
|
|
command: |
|
|
docker run -d \
|
|
-p 4001:4000 \
|
|
-e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
--name schema-seed \
|
|
--add-host=host.docker.internal:host-gateway \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--use_prisma_db_push
|
|
- wait_for_service:
|
|
url: http://localhost:4001
|
|
timeout: "300"
|
|
- run:
|
|
name: Stop schema seed container
|
|
command: docker stop schema-seed && docker rm schema-seed
|
|
- run:
|
|
name: Run Docker container with bad schema and disabled updates
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
|
|
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
|
|
-e DISABLE_SCHEMA_UPDATE="True" \
|
|
--name my-app \
|
|
--add-host=host.docker.internal:host-gateway \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "60"
|
|
- run:
|
|
name: Check container logs for expected message
|
|
command: |
|
|
echo "=== Printing Full Container Startup Logs ==="
|
|
LOG_OUTPUT="$(docker logs my-app 2>&1)"
|
|
printf '%s\n' "$LOG_OUTPUT"
|
|
echo "=== End of Full Container Startup Logs ==="
|
|
|
|
if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then
|
|
echo "Expected message found in logs. Test passed."
|
|
else
|
|
echo "Expected message not found in logs. Test failed."
|
|
exit 1
|
|
fi
|
|
- run:
|
|
name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion)
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit-2.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
build_and_test:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker tag litellm-docker-database:ci my-app:latest
|
|
- run:
|
|
name: Run Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e USE_PRISMA_MIGRATE=True \
|
|
-e AZURE_API_KEY=$AZURE_API_KEY \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
|
|
-e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
|
|
-e MISTRAL_API_KEY=$MISTRAL_API_KEY \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e GROQ_API_KEY=$GROQ_API_KEY \
|
|
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e AUTO_INFER_REGION=True \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
|
|
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
|
|
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
|
|
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/proxy_server_config.yaml:/app/config.yaml \
|
|
my-app:latest \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
# Original used `tests/*.py` (top-level only); the `--ignore=...`
|
|
# flags were vestigial since shell globbing did not descend into
|
|
# subdirectories. Replicate by globbing only top-level test files.
|
|
TEST_FILES=$(circleci tests glob "tests/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-s -v -x \
|
|
--junitxml=test-results/junit.xml \
|
|
-n 4 \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
e2e_openai_endpoints:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e AZURE_API_KEY=$AZURE_API_KEY \
|
|
-e AZURE_API_BASE=$AZURE_API_BASE \
|
|
-e AZURE_API_VERSION="2024-05-01-preview" \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
|
|
-e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
|
|
-e MISTRAL_API_KEY=$MISTRAL_API_KEY \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e GROQ_API_KEY=$GROQ_API_KEY \
|
|
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e AUTO_INFER_REGION=True \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
|
|
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
|
|
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
|
|
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-s -vv \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
proxy_logging_guardrails_model_info_tests:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
# intentionally give bad redis credentials here
|
|
# the OTEL test - should get this as a trace
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e OTEL_EXPORTER="in_memory" \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
|
-e GCS_FLUSH_INTERVAL="1" \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
# Clean up first container
|
|
- run:
|
|
name: Stop and remove first container
|
|
command: |
|
|
docker stop my-app
|
|
docker rm my-app
|
|
|
|
# Second Docker Container Run with Different Config
|
|
# NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license
|
|
- run:
|
|
name: Run Second Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE="bad-license" \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app-3 \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug
|
|
|
|
- run:
|
|
name: Start outputting logs for second container
|
|
command: docker logs -f my-app-3
|
|
background: true
|
|
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
|
|
- run:
|
|
name: Run second round of tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit-2.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
proxy_spend_accuracy_tests:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- start_redis
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
# Point the proxy at the job-local Redis (start_redis) instead of the
|
|
# shared remote Redis. The Redis transaction buffer uses a single
|
|
# global pod-lock key (cronjob_lock:db_spend_update_job) and a single
|
|
# global buffer list (litellm_spend_update_buffer); sharing those
|
|
# across concurrent CI pipelines causes spend flushes to stall or
|
|
# land in the wrong DB, which is what makes this test flaky.
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=host.docker.internal \
|
|
-e REDIS_PORT=6379 \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e PROXY_BATCH_WRITE_AT=2 \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: test-results
|
|
- run:
|
|
name: Stop and remove first container
|
|
when: always
|
|
command: |
|
|
docker stop my-app
|
|
docker rm my-app
|
|
docker stop redis-cache
|
|
docker rm redis-cache
|
|
|
|
proxy_multi_instance_tests:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container 1
|
|
# intentionally give bad redis credentials here
|
|
# the OTEL test - should get this as a trace
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Run Docker container 2
|
|
command: |
|
|
docker run -d \
|
|
-p 4001:4001 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app-2 \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4001 \
|
|
--detailed_debug
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- wait_for_service:
|
|
url: http://localhost:4001
|
|
timeout: "300"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
# Clean up first container
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
proxy_store_model_in_db_tests:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
# intentionally give bad redis credentials here
|
|
# the OTEL test - should get this as a trace
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e STORE_MODEL_IN_DB="True" \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Stop and remove containers
|
|
command: |
|
|
docker stop my-app || true
|
|
docker rm my-app || true
|
|
docker stop postgres-db || true
|
|
docker rm postgres-db || true
|
|
when: always
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
proxy_build_from_pip_tests:
|
|
# Change from docker to machine executor
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
# Remove Docker CLI installation since it's already available in machine executor
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- run:
|
|
name: Build Docker image
|
|
command: |
|
|
docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip .
|
|
- start_postgres
|
|
- run:
|
|
name: Run Docker container
|
|
# intentionally give bad redis credentials here
|
|
# the OTEL test - should get this as a trace
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e OTEL_EXPORTER="in_memory" \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e GCS_FLUSH_INTERVAL="1" \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \
|
|
my-app:latest \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x \
|
|
--junitxml=test-results/junit-2.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
# Clean up first container
|
|
- store_test_results:
|
|
path: test-results
|
|
- run:
|
|
name: Stop and remove first container
|
|
command: |
|
|
docker stop my-app || true
|
|
docker rm my-app || true
|
|
docker stop postgres-db || true
|
|
docker rm postgres-db || true
|
|
when: always
|
|
proxy_pass_through_endpoint_tests:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e GEMINI_API_KEY=$GEMINI_API_KEY \
|
|
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
|
-e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \
|
|
-e AZURE_API_KEY=$AZURE_API_KEY \
|
|
-e AZURE_API_BASE=$AZURE_API_BASE \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
# Add Ruby installation and testing before the existing Node.js and Python tests
|
|
- run:
|
|
name: Install Ruby and Bundler
|
|
command: |
|
|
# Clone RVM at pinned tag and verify the commit SHA matches the
|
|
# published tag before running its install script.
|
|
RVM_VERSION="1.29.12"
|
|
RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81"
|
|
git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm
|
|
RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)"
|
|
if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then
|
|
echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Import RVM signing keys (used by `rvm install` to verify Ruby tarballs)
|
|
gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
|
|
|
|
# Install RVM from the verified checkout. The install script
|
|
# sources `scripts/functions/installer` using paths relative to
|
|
# its own working directory, so it must be run from /tmp/rvm.
|
|
(cd /tmp/rvm && ./install --path "$HOME/.rvm")
|
|
source "$HOME/.rvm/scripts/rvm"
|
|
|
|
# Install Ruby 3.2.2 (RVM verifies the tarball PGP signature)
|
|
rvm install 3.2.2
|
|
rvm use 3.2.2 --default
|
|
|
|
# Install latest Bundler
|
|
gem install bundler
|
|
|
|
- run:
|
|
name: Run Ruby tests
|
|
command: |
|
|
source $HOME/.rvm/scripts/rvm
|
|
cd tests/pass_through_tests/ruby_passthrough_tests
|
|
bundle install
|
|
bundle exec rspec
|
|
no_output_timeout: 30m
|
|
# Install Node.js directly from nodejs.org with SHA256 verification,
|
|
# instead of piping NodeSource's setup_18.x apt-repo installer into
|
|
# sudo bash (which runs a mutable upstream script unattended).
|
|
- run:
|
|
name: Install Node.js 18.20.8
|
|
command: |
|
|
NODE_VERSION="18.20.8"
|
|
NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz"
|
|
NODE_EXPECTED_SHA="5467ee62d6af1411d46b6a10e3fb5cacc92734dbcef465fea14e7b90993001c9"
|
|
curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}"
|
|
echo "${NODE_EXPECTED_SHA} /tmp/${NODE_TARBALL}" | sha256sum -c -
|
|
sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /usr/local --strip-components=1
|
|
rm -f "/tmp/${NODE_TARBALL}"
|
|
node --version
|
|
npm --version
|
|
|
|
- run:
|
|
name: Install Node.js test dependencies
|
|
command: |
|
|
cd tests/pass_through_tests
|
|
npm ci
|
|
|
|
- run:
|
|
name: Run Vertex AI, Google AI Studio Node.js tests
|
|
command: |
|
|
cd tests/pass_through_tests
|
|
NODE_OPTIONS=--experimental-vm-modules npx jest . --verbose
|
|
no_output_timeout: 30m
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
proxy_e2e_anthropic_messages_tests:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container with test config
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e AWS_REGION_NAME="us-east-1" \
|
|
-e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- run:
|
|
name: Run Claude Agent SDK E2E Tests
|
|
command: |
|
|
mkdir -p test-results
|
|
export LITELLM_PROXY_URL="http://localhost:4000"
|
|
export LITELLM_API_KEY="sk-1234"
|
|
TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x -s \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
upload-coverage:
|
|
docker:
|
|
- *python312_image
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: .
|
|
# Check file locations
|
|
- run:
|
|
name: Check coverage file location
|
|
command: |
|
|
echo "Current directory:"
|
|
ls -la
|
|
echo "\nContents of tests/llm_translation:"
|
|
ls -la tests/llm_translation
|
|
- install_uv
|
|
- run:
|
|
name: Combine Coverage
|
|
command: |
|
|
uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage
|
|
uv tool run --from 'coverage[toml]==7.10.6' coverage xml
|
|
- codecov/upload:
|
|
file: ./coverage.xml
|
|
flags: circleci
|
|
|
|
ui_build:
|
|
docker:
|
|
- image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
resource_class: medium+
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
- ui-build-deps-v1-
|
|
- restore_cache:
|
|
keys:
|
|
- ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
- ui-nextjs-cache-v1-
|
|
- run:
|
|
name: Install dependencies
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm ci
|
|
- save_cache:
|
|
key: ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
paths:
|
|
- ui/litellm-dashboard/node_modules
|
|
- run:
|
|
name: Build UI
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
source ./build_ui.sh
|
|
- save_cache:
|
|
key: ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
paths:
|
|
- ui/litellm-dashboard/.next/cache
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- litellm/proxy/_experimental/out
|
|
|
|
ui_unit_tests:
|
|
docker:
|
|
- image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
resource_class: xlarge
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
- ui-unit-deps-v1-
|
|
- run:
|
|
name: Install dependencies
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm ci
|
|
- save_cache:
|
|
key: ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
paths:
|
|
- ui/litellm-dashboard/node_modules
|
|
- run:
|
|
name: Run UI unit tests (Vitest)
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
|
|
CI=true npm run test -- --run \
|
|
--pool forks --poolOptions.forks.maxForks=8
|
|
|
|
e2e_ui_testing:
|
|
docker:
|
|
- image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
|
|
environment:
|
|
POSTGRES_USER: e2euser
|
|
POSTGRES_PASSWORD: e2epassword
|
|
POSTGRES_DB: litellm_e2e
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
environment:
|
|
DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
|
|
CI: "true"
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Python dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma
|
|
- save_cache:
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
paths:
|
|
- ~/.cache/uv
|
|
- restore_cache:
|
|
keys:
|
|
- ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
- run:
|
|
name: Install Node dependencies and Playwright
|
|
# The cimg/python:3.12-browsers image already ships the Chromium system
|
|
# libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.).
|
|
# `--with-deps` triggers a redundant apt-get update + install that adds
|
|
# 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors,
|
|
# so we install just the browser binary.
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm ci
|
|
npx playwright install chromium
|
|
- save_cache:
|
|
key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
paths:
|
|
- ui/litellm-dashboard/node_modules
|
|
- ~/.cache/ms-playwright
|
|
- run:
|
|
name: Build UI from source
|
|
# Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`.
|
|
# GNU cp (used on CircleCI's Ubuntu image) interprets that as "copy the
|
|
# source directory as a child of the destination" when the destination
|
|
# already exists — silently creating `_experimental/out/out/` instead of
|
|
# replacing the served bundle. The proxy continued serving whatever was
|
|
# checked into `_experimental/out/*`, so this job was effectively testing
|
|
# the pre-build bundle on every run. Replace-and-move guarantees the
|
|
# freshly built bundle is what the proxy actually serves.
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm run build
|
|
rm -rf ../../litellm/proxy/_experimental/out
|
|
mv out ../../litellm/proxy/_experimental/out
|
|
# Restructure HTML so extensionless routes work (login.html -> login/index.html)
|
|
find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do
|
|
d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html"
|
|
done
|
|
- wait_for_service:
|
|
url: tcp://localhost:5432
|
|
timeout: "30"
|
|
- run:
|
|
name: Push Prisma schema
|
|
command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
|
|
- run:
|
|
name: Seed database
|
|
command: |
|
|
PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
|
|
-f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
|
|
- run:
|
|
name: Start mock LLM server
|
|
command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
|
|
background: true
|
|
- run:
|
|
name: Start LiteLLM proxy
|
|
environment:
|
|
LITELLM_MASTER_KEY: "sk-1234"
|
|
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
|
|
DISABLE_SCHEMA_UPDATE: "true"
|
|
SERVER_ROOT_PATH: ""
|
|
PROXY_LOGOUT_URL: ""
|
|
command: |
|
|
uv run --no-sync python -m litellm.proxy.proxy_cli \
|
|
--config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
|
|
--port 4000
|
|
background: true
|
|
- run:
|
|
name: Wait for proxy to be ready
|
|
command: |
|
|
for i in $(seq 1 60); do
|
|
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true)
|
|
if [ "$HTTP_CODE" = "200" ]; then
|
|
echo "Proxy is ready"
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "Proxy failed to start"
|
|
exit 1
|
|
- run:
|
|
name: Run Playwright E2E tests
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npx playwright test --config e2e_tests/playwright.config.ts
|
|
no_output_timeout: 10m
|
|
- store_artifacts:
|
|
path: ui/litellm-dashboard/test-results
|
|
destination: e2e-test-results
|
|
- store_artifacts:
|
|
path: ui/litellm-dashboard/playwright-report
|
|
destination: e2e-playwright-report
|
|
|
|
build_docker_database_image:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
|
|
- run:
|
|
name: Build Docker image
|
|
command: |
|
|
docker build \
|
|
-t litellm-docker-database:ci \
|
|
-f docker/Dockerfile.database .
|
|
|
|
- run:
|
|
name: Save Docker image to workspace root
|
|
command: |
|
|
docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst
|
|
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- litellm-docker-database.tar.zst
|
|
|
|
|
|
test_bad_database_url:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: medium
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- setup_google_dns
|
|
- start_postgres
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker tag litellm-docker-database:ci myapp:latest
|
|
- run:
|
|
name: Run Docker container with bad DATABASE_URL
|
|
command: |
|
|
docker run --name my-app \
|
|
-p 4000:4000 \
|
|
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
|
|
-e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
|
|
myapp:latest \
|
|
--port 4000 > docker_output.log 2>&1 || true
|
|
- run:
|
|
name: Display Docker logs
|
|
command: cat docker_output.log
|
|
- run:
|
|
name: Check for expected error
|
|
command: |
|
|
if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \
|
|
(grep -q "Database setup failed after multiple retries" docker_output.log || \
|
|
grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then
|
|
echo "Expected error found. Test passed."
|
|
else
|
|
echo "Expected error not found. Test failed."
|
|
cat docker_output.log
|
|
exit 1
|
|
fi
|
|
|
|
workflows:
|
|
build_and_test:
|
|
jobs:
|
|
- using_litellm_on_windows:
|
|
filters: &main_branches
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- local_testing_part1:
|
|
filters: *main_branches
|
|
- local_testing_part2:
|
|
filters: *main_branches
|
|
- langfuse_logging_unit_tests:
|
|
filters: *main_branches
|
|
- litellm_assistants_api_testing:
|
|
filters: *main_branches
|
|
- litellm_router_testing:
|
|
filters: *main_branches
|
|
- litellm_router_unit_testing:
|
|
filters: *main_branches
|
|
- ui_build:
|
|
filters: *main_branches
|
|
- ui_unit_tests:
|
|
requires:
|
|
- ui_build
|
|
filters: *main_branches
|
|
- auth_ui_unit_tests:
|
|
filters: *main_branches
|
|
- build_docker_database_image:
|
|
filters: *main_branches
|
|
- e2e_ui_testing:
|
|
filters: *main_branches
|
|
- build_and_test:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- e2e_openai_endpoints:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- proxy_logging_guardrails_model_info_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- proxy_spend_accuracy_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- proxy_multi_instance_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- proxy_store_model_in_db_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- proxy_build_from_pip_tests:
|
|
filters: *main_branches
|
|
- proxy_pass_through_endpoint_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- proxy_e2e_anthropic_messages_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- llm_translation_testing:
|
|
filters: *main_branches
|
|
- realtime_translation_testing:
|
|
filters: *main_branches
|
|
- agent_testing:
|
|
filters: *main_branches
|
|
- guardrails_testing:
|
|
filters: *main_branches
|
|
- google_generate_content_endpoint_testing:
|
|
filters: *main_branches
|
|
- llm_responses_api_testing:
|
|
filters: *main_branches
|
|
- ocr_testing:
|
|
filters: *main_branches
|
|
- search_testing:
|
|
filters: *main_branches
|
|
- litellm_mapped_enterprise_tests:
|
|
filters: *main_branches
|
|
- batches_testing:
|
|
filters: *main_branches
|
|
- litellm_utils_testing:
|
|
filters: *main_branches
|
|
- pass_through_unit_testing:
|
|
filters: *main_branches
|
|
- image_gen_testing:
|
|
filters: *main_branches
|
|
- logging_testing:
|
|
filters: *main_branches
|
|
- audio_testing:
|
|
filters: *main_branches
|
|
- redis_caching_unit_tests:
|
|
filters: *main_branches
|
|
- upload-coverage:
|
|
requires:
|
|
- realtime_translation_testing
|
|
- agent_testing
|
|
- google_generate_content_endpoint_testing
|
|
- guardrails_testing
|
|
- ocr_testing
|
|
- search_testing
|
|
- litellm_mapped_enterprise_tests
|
|
- batches_testing
|
|
- litellm_utils_testing
|
|
- pass_through_unit_testing
|
|
- image_gen_testing
|
|
- logging_testing
|
|
- audio_testing
|
|
- redis_caching_unit_tests
|
|
- langfuse_logging_unit_tests
|
|
- local_testing_part1
|
|
- local_testing_part2
|
|
- litellm_assistants_api_testing
|
|
- litellm_router_unit_testing
|
|
- auth_ui_unit_tests
|
|
- db_migration_disable_update_check:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- installing_litellm_on_python:
|
|
filters: *main_branches
|
|
- installing_litellm_on_python_3_13:
|
|
filters: *main_branches
|
|
- installing_litellm_on_python_v2_migration_resolver:
|
|
filters: *main_branches
|
|
- helm_chart_testing:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- test_bad_database_url:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|