a6c30b30bf
* build: migrate packaging metadata to uv * ci: move automation and local tooling to uv * docker: migrate image builds and runtime setup to uv * docs: update install and deployment guidance for uv * chore: align auxiliary scripts and tests with uv * test: harden test_litellm isolation * fix: keep release and health check images self-contained * build: pin uv tooling and health check deps * test: isolate bedrock image request formatting from suite state * test: cover sandbox executor requirements flow * ci: fix circleci no-op command steps * ci: fix circleci publish workflow parsing * fix: stabilize remaining uv migration CI checks * ci: increase matrix test timeout headroom * fix: restore published docker and license coverage * fix: restore proxy runtime build parity * fix: restore proxy extras parity and venv migrations * ci: persist uv path across circleci steps * fix: keep psycopg binary in default test env * docker: preserve prisma cache across stages * test: run local proxy checks through uv python * build: restore runtime deps moved into ci * build: refresh uv lock after upstream merge * fix: restore module import in test_check_migration after merge The conflict resolution imported only the function but the test body references check_migration as a module throughout. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: revert dependency promotions, remove nodejs-wheel-binaries, fix Docker layer caching - Move google-generativeai, Pillow, tenacity back to ci group (they are lazily imported and bloat the base SDK install needlessly) - Remove nodejs-wheel-binaries from extra_proxy and proxy-dev (redundant in Docker where system Node.js is already installed via apk) - Remove all nodejs-wheel node replacement and venv npm patching blocks from Dockerfiles since the wheel is no longer installed - Add --no-default-groups to CodSpeed benchmark workflow so the benchmark environment matches the old minimal pip install footprint - Apply standard uv two-phase Docker pattern: copy metadata first, install deps (cached layer), then copy source and install project - Replace CircleCI enterprise no-op with proper uv sync command Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * chore: regenerate uv.lock after removing nodejs-wheel-binaries Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(ci): use cache/restore instead of cache to prevent cache poisoning The old workflow used actions/cache/restore (read-only). The uv migration changed it to actions/cache (read-write), which zizmor flags as a cache poisoning risk. Restore the safer read-only variant. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(ci): disable setup-uv built-in cache to silence cache-poisoning alert The setup-uv action enables caching by default, which zizmor flags as a cache poisoning risk. Disable it since we already use a read-only cache/restore step. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(ci): disable setup-uv cache in publish workflow Silences zizmor cache-poisoning alert. Publishing workflow runs infrequently on protected branches so caching adds no real benefit. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(test): remove duplicate verbose_logger mock in test_check_migration The logger was patched twice — first via mocker.patch() then via mocker.patch.object(autospec=True). The second call fails because autospec cannot inspect an already-mocked attribute. Remove the redundant first patch. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(ci): free disk space before Docker build in test-server-root-path The Dockerfile.non_root build ran out of disk on the CI runner. Remove Android SDK, .NET, Boost, and GHC toolchains (~12GB) to free space. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
3748 lines
141 KiB
YAML
3748 lines
141 KiB
YAML
version: 2.1
|
|
orbs:
|
|
codecov: codecov/codecov@4.0.1
|
|
node: circleci/node@5.1.0 # Add this line to declare the node orb
|
|
win: circleci/windows@5.0 # Add Windows orb
|
|
|
|
commands:
|
|
setup_google_dns:
|
|
steps:
|
|
- run:
|
|
name: "Configure Google DNS"
|
|
command: |
|
|
# Backup original resolv.conf
|
|
sudo cp /etc/resolv.conf /etc/resolv.conf.backup
|
|
# Add both local and Google DNS servers
|
|
echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf
|
|
echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf
|
|
echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf
|
|
setup_litellm_enterprise_pip:
|
|
steps:
|
|
- run:
|
|
name: "Install local version of litellm-enterprise"
|
|
command: uv sync --frozen --package litellm-enterprise --python "$(which python)"
|
|
setup_litellm_test_deps:
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- v3-litellm-uv-deps-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- setup_litellm_enterprise_pip
|
|
- save_cache:
|
|
paths:
|
|
- ~/.local/lib
|
|
- ~/.local/bin
|
|
- ~/.cache/uv
|
|
key: v3-litellm-uv-deps-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }}
|
|
|
|
jobs:
|
|
# Add Windows testing job
|
|
using_litellm_on_windows:
|
|
executor:
|
|
name: win/default
|
|
shell: powershell.exe
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install Python
|
|
command: |
|
|
choco install python --version=3.11.0 -y --no-progress --force
|
|
refreshenv
|
|
python --version
|
|
environment:
|
|
CHOCOLATEY_CONFIRM_ALL: "true"
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
Invoke-RestMethod https://astral.sh/uv/0.10.9/install.ps1 | Invoke-Expression
|
|
$uvBin = Join-Path $HOME ".local\bin"
|
|
$env:Path = "$uvBin;$env:Path"
|
|
if (!(Test-Path $PROFILE)) {
|
|
New-Item -ItemType File -Force -Path $PROFILE | Out-Null
|
|
}
|
|
if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) {
|
|
Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`""
|
|
}
|
|
uv sync --frozen --group dev --python (Get-Command python).Source
|
|
- run:
|
|
name: Run Windows-specific test
|
|
command: |
|
|
uv run --no-sync python -m pytest tests/windows_tests/test_litellm_on_windows.py -v
|
|
|
|
mypy_linting:
|
|
docker:
|
|
- image: cimg/python:3.12
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: medium
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
uv sync --frozen --group dev --python "$(which python)" --no-install-package fastuuid
|
|
- run:
|
|
name: MyPy Type Checking
|
|
command: |
|
|
cd litellm
|
|
# Use the same approach as GitHub Actions, explicitly exclude fastuuid to avoid segfaults
|
|
uv run --no-sync python -m mypy .
|
|
cd ..
|
|
no_output_timeout: 10m
|
|
|
|
semgrep:
|
|
docker:
|
|
- image: cimg/python:3.12
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: medium
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Semgrep
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
- run:
|
|
name: Run Semgrep (custom rules only)
|
|
command: |
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
uv tool run --from 'semgrep==1.157.0' semgrep scan --config .semgrep/rules . --error
|
|
|
|
local_testing_part1:
|
|
docker:
|
|
- image: cimg/python:3.12
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
parallelism: 4
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Show git commit hash
|
|
command: |
|
|
echo "Git commit hash: $CIRCLE_SHA1"
|
|
|
|
- restore_cache:
|
|
keys:
|
|
- v1-dependencies-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- setup_litellm_enterprise_pip
|
|
- save_cache:
|
|
paths:
|
|
- ./.venv
|
|
key: v1-dependencies-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Run prisma ./docker/entrypoint.sh
|
|
command: |
|
|
set +e
|
|
chmod +x docker/entrypoint.sh
|
|
./docker/entrypoint.sh
|
|
set -e
|
|
- run:
|
|
name: Black Formatting
|
|
command: |
|
|
cd litellm
|
|
uv run --no-sync python -m black .
|
|
cd ..
|
|
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests (Part 1 - A-M)
|
|
command: |
|
|
mkdir test-results
|
|
|
|
# Discover test files (A-M)
|
|
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py")
|
|
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--split-by=timings \
|
|
--verbose \
|
|
--command="xargs uv run --no-sync python -m pytest \
|
|
-vv \
|
|
--cov=litellm \
|
|
--cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=20 \
|
|
-k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
|
|
-n 4 \
|
|
--timeout=300 \
|
|
--timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml local_testing_part1_coverage.xml
|
|
mv .coverage local_testing_part1_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- local_testing_part1_coverage.xml
|
|
- local_testing_part1_coverage
|
|
local_testing_part2:
|
|
docker:
|
|
- image: cimg/python:3.12
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
parallelism: 4
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Show git commit hash
|
|
command: |
|
|
echo "Git commit hash: $CIRCLE_SHA1"
|
|
|
|
- restore_cache:
|
|
keys:
|
|
- v1-dependencies-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- setup_litellm_enterprise_pip
|
|
- save_cache:
|
|
paths:
|
|
- ./.venv
|
|
key: v1-dependencies-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Run prisma ./docker/entrypoint.sh
|
|
command: |
|
|
set +e
|
|
chmod +x docker/entrypoint.sh
|
|
./docker/entrypoint.sh
|
|
set -e
|
|
- run:
|
|
name: Black Formatting
|
|
command: |
|
|
cd litellm
|
|
uv run --no-sync python -m black .
|
|
cd ..
|
|
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests (Part 2 - N-Z)
|
|
command: |
|
|
mkdir test-results
|
|
|
|
# Discover test files (N-Z)
|
|
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py")
|
|
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--split-by=timings \
|
|
--verbose \
|
|
--command="xargs uv run --no-sync python -m pytest \
|
|
-vv \
|
|
--cov=litellm \
|
|
--cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=20 \
|
|
-k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
|
|
-n 4 \
|
|
--timeout=300 \
|
|
--timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml local_testing_part2_coverage.xml
|
|
mv .coverage local_testing_part2_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- local_testing_part2_coverage.xml
|
|
- local_testing_part2_coverage
|
|
langfuse_logging_unit_tests:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: medium
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Show git commit hash
|
|
command: |
|
|
echo "Git commit hash: $CIRCLE_SHA1"
|
|
|
|
- restore_cache:
|
|
keys:
|
|
- v1-dependencies-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- setup_litellm_enterprise_pip
|
|
- save_cache:
|
|
paths:
|
|
- ./.venv
|
|
key: v1-dependencies-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Run prisma ./docker/entrypoint.sh
|
|
command: |
|
|
set +e
|
|
chmod +x docker/entrypoint.sh
|
|
./docker/entrypoint.sh
|
|
set -e
|
|
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse"
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
auth_ui_unit_tests:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- save_cache:
|
|
paths:
|
|
- ./.venv
|
|
key: v1-dependencies-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Run prisma ./docker/entrypoint.sh
|
|
command: |
|
|
set +e
|
|
chmod +x docker/entrypoint.sh
|
|
./docker/entrypoint.sh
|
|
set -e
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
litellm_router_testing: # Runs all tests with the "router" keyword
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
parallelism: 4
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- v1-router-testing-deps-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- save_cache:
|
|
paths:
|
|
- /home/circleci/.pyenv
|
|
- /home/circleci/.local
|
|
key: v1-router-testing-deps-{{ checksum "uv.lock" }}
|
|
# Run pytest and generate JUnit XML report
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
|
|
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--split-by=timings \
|
|
--verbose \
|
|
--command="xargs uv run --no-sync python -m pytest \
|
|
-v \
|
|
-k 'router' \
|
|
-n 4 \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
--timeout=300 --timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
litellm_router_unit_testing: # Runs all tests with the "router" keyword
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- v1-router-unit-deps-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- save_cache:
|
|
paths:
|
|
- /home/circleci/.pyenv
|
|
- /home/circleci/.local
|
|
key: v1-router-unit-deps-{{ checksum "uv.lock" }}
|
|
# Run pytest and generate JUnit XML report
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
|
|
docker:
|
|
- image: cimg/python:3.13.1
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: medium
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
# Run pytest and generate JUnit XML report
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
llm_translation_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- v1-llm-translation-deps-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- save_cache:
|
|
paths:
|
|
- /home/circleci/.pyenv
|
|
- /home/circleci/.local
|
|
key: v1-llm-translation-deps-{{ checksum "uv.lock" }}
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
# Add --timeout to kill hanging tests after 120s (2 min)
|
|
# Add --durations=20 to show 20 slowest tests for debugging
|
|
# Subdirectories with dedicated jobs (maintain this list as new jobs are added)
|
|
IGNORE_DIRS=(
|
|
"tests/llm_translation/realtime"
|
|
)
|
|
IGNORE_ARGS=""
|
|
for dir in "${IGNORE_DIRS[@]}"; do
|
|
IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir"
|
|
done
|
|
uv run --no-sync python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
realtime_translation_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run realtime tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
# Add --timeout to kill hanging tests after 120s (2 min)
|
|
# Add --durations=20 to show 20 slowest tests for debugging
|
|
uv run --no-sync python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml realtime_translation_coverage.xml
|
|
mv .coverage realtime_translation_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- realtime_translation_coverage.xml
|
|
- realtime_translation_coverage
|
|
mcp_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -vv tests/mcp_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml mcp_coverage.xml
|
|
mv .coverage mcp_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- mcp_coverage.xml
|
|
- mcp_coverage
|
|
agent_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml agent_coverage.xml
|
|
mv .coverage agent_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- agent_coverage.xml
|
|
- agent_coverage
|
|
guardrails_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml guardrails_coverage.xml
|
|
mv .coverage guardrails_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- guardrails_coverage.xml
|
|
- guardrails_coverage
|
|
|
|
google_generate_content_endpoint_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml google_generate_content_endpoint_coverage.xml
|
|
mv .coverage google_generate_content_endpoint_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- google_generate_content_endpoint_coverage.xml
|
|
- google_generate_content_endpoint_coverage
|
|
|
|
llm_responses_api_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- v1-llm-responses-deps-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- save_cache:
|
|
paths:
|
|
- /home/circleci/.pyenv
|
|
- /home/circleci/.local
|
|
key: v1-llm-responses-deps-{{ checksum "uv.lock" }}
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
ocr_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml ocr_coverage.xml
|
|
mv .coverage ocr_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- ocr_coverage.xml
|
|
- ocr_coverage
|
|
search_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml search_coverage.xml
|
|
mv .coverage search_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- search_coverage.xml
|
|
- search_coverage
|
|
# Split litellm_mapped_tests into parallel jobs
|
|
litellm_mapped_tests_proxy_part1:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
steps:
|
|
- setup_litellm_test_deps
|
|
- run:
|
|
name: Run proxy tests part 1 (high-volume directories)
|
|
command: |
|
|
prisma generate
|
|
export PYTHONUNBUFFERED=1
|
|
uv run --no-sync python -m pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/client tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part1.xml --durations=10 -n 4 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: test-results
|
|
litellm_mapped_tests_proxy_part2:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
steps:
|
|
- setup_litellm_test_deps
|
|
- run:
|
|
name: Run proxy tests part 2 (all other tests)
|
|
command: |
|
|
prisma generate
|
|
export PYTHONUNBUFFERED=1
|
|
uv run --no-sync python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: test-results
|
|
litellm_mapped_enterprise_tests:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run enterprise tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
prisma generate
|
|
uv run --no-sync python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
batches_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml batches_coverage.xml
|
|
mv .coverage batches_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- batches_coverage.xml
|
|
- batches_coverage
|
|
litellm_utils_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml litellm_utils_coverage.xml
|
|
mv .coverage litellm_utils_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- litellm_utils_coverage.xml
|
|
- litellm_utils_coverage
|
|
|
|
pass_through_unit_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml pass_through_unit_tests_coverage.xml
|
|
mv .coverage pass_through_unit_tests_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- pass_through_unit_tests_coverage.xml
|
|
- pass_through_unit_tests_coverage
|
|
image_gen_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
logging_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
# Run pytest and generate JUnit XML report
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml logging_coverage.xml || true
|
|
mv .coverage logging_coverage || true
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- logging_coverage.xml
|
|
- logging_coverage
|
|
audio_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml audio_coverage.xml
|
|
mv .coverage audio_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- audio_coverage.xml
|
|
- audio_coverage
|
|
redis_caching_unit_tests:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
python -m pip install --upgrade pip uv
|
|
uv pip install --system -r requirements.txt
|
|
pip install "pytest==7.3.1"
|
|
pip install "pytest-retry==1.6.3"
|
|
pip install "pytest-cov==5.0.0"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
pip install "pytest-xdist==3.6.1"
|
|
pip install "pytest-rerunfailures==14.0"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
python -m pytest -vv \
|
|
tests/local_testing/test_dual_cache.py \
|
|
tests/local_testing/test_redis_batch_optimizations.py \
|
|
tests/local_testing/test_router_utils.py \
|
|
--cov=litellm --cov-report=xml \
|
|
-x -s -v --junitxml=test-results/junit.xml \
|
|
--durations=5 -n 2 \
|
|
--reruns 2 --reruns-delay 1
|
|
no_output_timeout: 20m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml redis_caching_coverage.xml
|
|
mv .coverage redis_caching_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- redis_caching_coverage.xml
|
|
- redis_caching_coverage
|
|
installing_litellm_on_python:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py
|
|
|
|
installing_litellm_on_python_3_13:
|
|
docker:
|
|
- image: cimg/python:3.13.1
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: medium
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py
|
|
helm_chart_testing:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker
|
|
resource_class: medium
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- setup_google_dns
|
|
# Install Helm
|
|
- run:
|
|
name: Install Helm
|
|
command: |
|
|
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
|
|
|
|
# Install kind
|
|
- run:
|
|
name: Install Kind
|
|
command: |
|
|
curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
|
|
chmod +x ./kind
|
|
sudo mv ./kind /usr/local/bin/kind
|
|
|
|
# Install kubectl
|
|
- run:
|
|
name: Install kubectl
|
|
command: |
|
|
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
|
|
chmod +x kubectl
|
|
sudo mv kubectl /usr/local/bin/
|
|
|
|
# Create kind cluster
|
|
- run:
|
|
name: Create Kind Cluster
|
|
command: |
|
|
kind create cluster --name litellm-test
|
|
|
|
- run:
|
|
name: Load Docker Database Image for helm tests
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
|
docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG}
|
|
|
|
- run:
|
|
name: Load Docker image into Kind
|
|
command: |
|
|
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
|
kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test
|
|
|
|
# Run helm lint
|
|
- run:
|
|
name: Run helm lint
|
|
command: |
|
|
helm lint ./deploy/charts/litellm-helm
|
|
|
|
# Run helm tests
|
|
- run:
|
|
name: Run helm tests
|
|
command: |
|
|
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
|
helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \
|
|
--set image.repository=litellm-ci \
|
|
--set image.tag=${IMAGE_TAG} \
|
|
--set image.pullPolicy=Never
|
|
# Wait for pod to be ready
|
|
echo "Waiting 30 seconds for pod to be ready..."
|
|
sleep 30
|
|
|
|
# Print pod logs before running tests
|
|
echo "Printing pod logs..."
|
|
kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}")
|
|
|
|
# Run the helm tests
|
|
helm test litellm --logs
|
|
helm test litellm --logs
|
|
|
|
# Cleanup
|
|
- run:
|
|
name: Cleanup
|
|
command: |
|
|
kind delete cluster --name litellm-test
|
|
when: always # This ensures cleanup runs even if previous steps fail
|
|
|
|
check_code_and_doc_quality:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project/litellm
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- run: uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
|
|
- run: ruff check ./litellm
|
|
# - run: python ./tests/documentation_tests/test_general_setting_keys.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/info_log_check.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/test_ban_set_verbose.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/code_qa_check_tests.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/test_proxy_types_import.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/callback_manager_test.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/test_router_strategy_async.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/litellm_logging_code_coverage.py
|
|
- run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
|
|
- run: uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
|
|
- run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/ensure_async_clients_test.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/enforce_llms_folder_style.py
|
|
- run: uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py
|
|
- run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
|
|
- run: helm lint ./deploy/charts/litellm-helm
|
|
|
|
db_migration_disable_update_check:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: medium
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Python 3.9
|
|
command: |
|
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
|
bash miniconda.sh -b -p $HOME/miniconda
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
conda init bash
|
|
source ~/.bashrc
|
|
conda create -n myenv python=3.9 -y
|
|
conda activate myenv
|
|
python --version
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- run:
|
|
name: Install dockerize
|
|
command: |
|
|
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start PostgreSQL Database
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=litellm_test \
|
|
-p 5432:5432 \
|
|
postgres:14
|
|
- run:
|
|
name: Wait for PostgreSQL to be ready
|
|
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Seed database with real schema
|
|
command: |
|
|
docker run -d \
|
|
-p 4001:4000 \
|
|
-e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
--name schema-seed \
|
|
--add-host=host.docker.internal:host-gateway \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--use_prisma_db_push
|
|
- run:
|
|
name: Wait for schema seed to complete
|
|
command: dockerize -wait http://localhost:4001 -timeout 5m
|
|
- run:
|
|
name: Stop schema seed container
|
|
command: docker stop schema-seed && docker rm schema-seed
|
|
- run:
|
|
name: Run Docker container with bad schema and disabled updates
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
|
|
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
|
|
-e DISABLE_SCHEMA_UPDATE="True" \
|
|
--name my-app \
|
|
--add-host=host.docker.internal:host-gateway \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000
|
|
- run:
|
|
name: Wait for container to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 1m
|
|
- run:
|
|
name: Check container logs for expected message
|
|
command: |
|
|
echo "=== Printing Full Container Startup Logs ==="
|
|
LOG_OUTPUT="$(docker logs my-app 2>&1)"
|
|
printf '%s\n' "$LOG_OUTPUT"
|
|
echo "=== End of Full Container Startup Logs ==="
|
|
|
|
if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then
|
|
echo "Expected message found in logs. Test passed."
|
|
else
|
|
echo "Expected message not found in logs. Test failed."
|
|
exit 1
|
|
fi
|
|
- run:
|
|
name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion)
|
|
command: |
|
|
uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
|
|
no_output_timeout: 15m
|
|
|
|
build_and_test:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Python 3.9
|
|
command: |
|
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
|
bash miniconda.sh -b -p $HOME/miniconda
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
conda init bash
|
|
source ~/.bashrc
|
|
conda create -n myenv python=3.9 -y
|
|
conda activate myenv
|
|
python --version
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- run:
|
|
name: Install dockerize
|
|
command: |
|
|
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start PostgreSQL Database
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=circle_test \
|
|
-p 5432:5432 \
|
|
postgres:14
|
|
- run:
|
|
name: Wait for PostgreSQL to be ready
|
|
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker tag litellm-docker-database:ci my-app:latest
|
|
- run:
|
|
name: Run Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e USE_PRISMA_MIGRATE=True \
|
|
-e AZURE_API_KEY=$AZURE_API_KEY \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
|
|
-e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
|
|
-e MISTRAL_API_KEY=$MISTRAL_API_KEY \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e GROQ_API_KEY=$GROQ_API_KEY \
|
|
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e AUTO_INFER_REGION=True \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
|
|
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
|
|
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
|
|
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/proxy_server_config.yaml:/app/config.yaml \
|
|
my-app:latest \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Install curl
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y curl
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- run:
|
|
name: Wait for app to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
e2e_openai_endpoints:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Docker CLI (In case it's not already installed)
|
|
command: |
|
|
curl -fsSL https://get.docker.com | sh
|
|
sudo usermod -aG docker $USER
|
|
docker version
|
|
- run:
|
|
name: Install Python 3.10
|
|
command: |
|
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
|
bash miniconda.sh -b -p $HOME/miniconda
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
conda init bash
|
|
source ~/.bashrc
|
|
conda create -n myenv python=3.10 -y
|
|
conda activate myenv
|
|
python --version
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- run:
|
|
name: Install dockerize
|
|
command: |
|
|
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start PostgreSQL Database
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=circle_test \
|
|
-p 5432:5432 \
|
|
postgres:14
|
|
- run:
|
|
name: Wait for PostgreSQL to be ready
|
|
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e AZURE_API_KEY=$AZURE_API_KEY \
|
|
-e AZURE_API_BASE=$AZURE_API_BASE \
|
|
-e AZURE_API_VERSION="2024-05-01-preview" \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
|
|
-e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
|
|
-e MISTRAL_API_KEY=$MISTRAL_API_KEY \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e GROQ_API_KEY=$GROQ_API_KEY \
|
|
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e AUTO_INFER_REGION=True \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
|
|
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
|
|
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
|
|
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Install curl
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y curl
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- run:
|
|
name: Wait for app to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
proxy_logging_guardrails_model_info_tests:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Docker CLI (In case it's not already installed)
|
|
command: |
|
|
curl -fsSL https://get.docker.com | sh
|
|
sudo usermod -aG docker $USER
|
|
docker version
|
|
- run:
|
|
name: Install Python 3.9
|
|
command: |
|
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
|
bash miniconda.sh -b -p $HOME/miniconda
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
conda init bash
|
|
source ~/.bashrc
|
|
conda create -n myenv python=3.9 -y
|
|
conda activate myenv
|
|
python --version
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- run:
|
|
name: Install dockerize
|
|
command: |
|
|
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start PostgreSQL Database
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=circle_test \
|
|
-p 5432:5432 \
|
|
postgres:14
|
|
- run:
|
|
name: Wait for PostgreSQL to be ready
|
|
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
# intentionally give bad redis credentials here
|
|
# the OTEL test - should get this as a trace
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e OTEL_EXPORTER="in_memory" \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
|
-e GCS_FLUSH_INTERVAL="1" \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Install curl and dockerize
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y curl
|
|
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- run:
|
|
name: Wait for app to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 15m
|
|
# Clean up first container
|
|
- run:
|
|
name: Stop and remove first container
|
|
command: |
|
|
docker stop my-app
|
|
docker rm my-app
|
|
|
|
# Second Docker Container Run with Different Config
|
|
# NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license
|
|
- run:
|
|
name: Run Second Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE="bad-license" \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app-3 \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug
|
|
|
|
- run:
|
|
name: Start outputting logs for second container
|
|
command: docker logs -f my-app-2
|
|
background: true
|
|
|
|
- run:
|
|
name: Wait for second app to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
|
|
- run:
|
|
name: Run second round of tests
|
|
command: |
|
|
uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
proxy_spend_accuracy_tests:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Docker CLI (In case it's not already installed)
|
|
command: |
|
|
curl -fsSL https://get.docker.com | sh
|
|
sudo usermod -aG docker $USER
|
|
docker version
|
|
- run:
|
|
name: Install Python 3.9
|
|
command: |
|
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
|
bash miniconda.sh -b -p $HOME/miniconda
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
conda init bash
|
|
source ~/.bashrc
|
|
conda create -n myenv python=3.9 -y
|
|
conda activate myenv
|
|
python --version
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- run:
|
|
name: Install dockerize
|
|
command: |
|
|
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start PostgreSQL Database
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=circle_test \
|
|
-p 5432:5432 \
|
|
postgres:14
|
|
- run:
|
|
name: Wait for PostgreSQL to be ready
|
|
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
# intentionally give bad redis credentials here
|
|
# the OTEL test - should get this as a trace
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Install curl
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y curl
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- run:
|
|
name: Wait for app to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 15m
|
|
# Clean up first container
|
|
- run:
|
|
name: Stop and remove first container
|
|
command: |
|
|
docker stop my-app
|
|
docker rm my-app
|
|
|
|
proxy_multi_instance_tests:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Docker CLI (In case it's not already installed)
|
|
command: |
|
|
curl -fsSL https://get.docker.com | sh
|
|
sudo usermod -aG docker $USER
|
|
docker version
|
|
- run:
|
|
name: Install Python 3.9
|
|
command: |
|
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
|
bash miniconda.sh -b -p $HOME/miniconda
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
conda init bash
|
|
source ~/.bashrc
|
|
conda create -n myenv python=3.9 -y
|
|
conda activate myenv
|
|
python --version
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- run:
|
|
name: Install dockerize
|
|
command: |
|
|
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start PostgreSQL Database
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=circle_test \
|
|
-p 5432:5432 \
|
|
postgres:14
|
|
- run:
|
|
name: Wait for PostgreSQL to be ready
|
|
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container 1
|
|
# intentionally give bad redis credentials here
|
|
# the OTEL test - should get this as a trace
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Run Docker container 2
|
|
command: |
|
|
docker run -d \
|
|
-p 4001:4001 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app-2 \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4001 \
|
|
--detailed_debug
|
|
- run:
|
|
name: Install curl and dockerize
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y curl
|
|
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- run:
|
|
name: Wait for instance 1 to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
- run:
|
|
name: Wait for instance 2 to be ready
|
|
command: dockerize -wait http://localhost:4001 -timeout 5m
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 15m
|
|
# Clean up first container
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
proxy_store_model_in_db_tests:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Docker CLI (In case it's not already installed)
|
|
command: |
|
|
curl -fsSL https://get.docker.com | sh
|
|
sudo usermod -aG docker $USER
|
|
docker version
|
|
sudo systemctl restart docker
|
|
- run:
|
|
name: Install Python 3.9
|
|
command: |
|
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
|
bash miniconda.sh -b -p $HOME/miniconda
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
conda init bash
|
|
source ~/.bashrc
|
|
conda create -n myenv python=3.9 -y
|
|
conda activate myenv
|
|
python --version
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- run:
|
|
name: Install dockerize
|
|
command: |
|
|
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start PostgreSQL Database
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=circle_test \
|
|
-p 5432:5432 \
|
|
postgres:14
|
|
- run:
|
|
name: Wait for PostgreSQL to be ready
|
|
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
# intentionally give bad redis credentials here
|
|
# the OTEL test - should get this as a trace
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e STORE_MODEL_IN_DB="True" \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Install curl and dockerize
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y curl
|
|
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- run:
|
|
name: Wait for app to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Stop and remove containers
|
|
command: |
|
|
docker stop my-app || true
|
|
docker rm my-app || true
|
|
docker stop postgres-db || true
|
|
docker rm postgres-db || true
|
|
when: always
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
proxy_build_from_pip_tests:
|
|
# Change from docker to machine executor
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
# Remove Docker CLI installation since it's already available in machine executor
|
|
- run:
|
|
name: Install Python 3.13
|
|
command: |
|
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
|
bash miniconda.sh -b -p $HOME/miniconda
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
conda init bash
|
|
source ~/.bashrc
|
|
conda create -n myenv python=3.13 -y
|
|
conda activate myenv
|
|
python --version
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- run:
|
|
name: Build Docker image
|
|
command: |
|
|
docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip .
|
|
- run:
|
|
name: Start PostgreSQL Database
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=circle_test \
|
|
-p 5432:5432 \
|
|
postgres:14
|
|
- run:
|
|
name: Wait for PostgreSQL to be ready
|
|
command: |
|
|
timeout 60s bash -c 'until docker exec postgres-db pg_isready -U postgres -d circle_test; do sleep 2; done'
|
|
- run:
|
|
name: Run Docker container
|
|
# intentionally give bad redis credentials here
|
|
# the OTEL test - should get this as a trace
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e OTEL_EXPORTER="in_memory" \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e GCS_FLUSH_INTERVAL="1" \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \
|
|
my-app:latest \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Install curl and dockerize
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y curl
|
|
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- run:
|
|
name: Wait for app to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
uv run --no-sync python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
|
|
no_output_timeout: 15m
|
|
# Clean up first container
|
|
- run:
|
|
name: Stop and remove first container
|
|
command: |
|
|
docker stop my-app || true
|
|
docker rm my-app || true
|
|
docker stop postgres-db || true
|
|
docker rm postgres-db || true
|
|
when: always
|
|
proxy_pass_through_endpoint_tests:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Python 3.10
|
|
command: |
|
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
|
bash miniconda.sh -b -p $HOME/miniconda
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
conda init bash
|
|
source ~/.bashrc
|
|
conda create -n myenv python=3.10 -y
|
|
conda activate myenv
|
|
python --version
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- run:
|
|
name: Install dockerize
|
|
command: |
|
|
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start PostgreSQL Database
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=circle_test \
|
|
-p 5432:5432 \
|
|
postgres:14
|
|
- run:
|
|
name: Wait for PostgreSQL to be ready
|
|
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e GEMINI_API_KEY=$GEMINI_API_KEY \
|
|
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
|
-e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \
|
|
-e AZURE_API_KEY=$AZURE_API_KEY \
|
|
-e AZURE_API_BASE=$AZURE_API_BASE \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- run:
|
|
name: Wait for app to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
# Add Ruby installation and testing before the existing Node.js and Python tests
|
|
- run:
|
|
name: Install Ruby and Bundler
|
|
command: |
|
|
# Import GPG keys first
|
|
gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB || {
|
|
curl -sSL https://rvm.io/mpapis.asc | gpg --import -
|
|
curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -
|
|
}
|
|
|
|
# Install Ruby version manager (RVM)
|
|
curl -sSL https://get.rvm.io | bash -s stable
|
|
|
|
# Source RVM from the correct location
|
|
source $HOME/.rvm/scripts/rvm
|
|
|
|
# Install Ruby 3.2.2
|
|
rvm install 3.2.2
|
|
rvm use 3.2.2 --default
|
|
|
|
# Install latest Bundler
|
|
gem install bundler
|
|
|
|
- run:
|
|
name: Run Ruby tests
|
|
command: |
|
|
source $HOME/.rvm/scripts/rvm
|
|
cd tests/pass_through_tests/ruby_passthrough_tests
|
|
bundle install
|
|
bundle exec rspec
|
|
no_output_timeout: 30m
|
|
# New steps to run Node.js test
|
|
- run:
|
|
name: Install Node.js
|
|
command: |
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash -
|
|
sudo apt-get update
|
|
sudo apt-get install -y nodejs
|
|
node --version
|
|
npm --version
|
|
|
|
- run:
|
|
name: Install Node.js dependencies
|
|
command: |
|
|
npm install @google-cloud/vertexai
|
|
npm install @google/generative-ai
|
|
npm install --save-dev jest
|
|
|
|
- run:
|
|
name: Run Vertex AI, Google AI Studio Node.js tests
|
|
command: |
|
|
npx jest tests/pass_through_tests --verbose
|
|
no_output_timeout: 30m
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source $HOME/miniconda/etc/profile.d/conda.sh
|
|
conda activate myenv
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
proxy_e2e_anthropic_messages_tests:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install Docker CLI (In case it's not already installed)
|
|
command: |
|
|
curl -fsSL https://get.docker.com | sh
|
|
sudo usermod -aG docker $USER
|
|
docker version
|
|
- run:
|
|
name: Install Python 3.10
|
|
command: |
|
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
|
bash miniconda.sh -b -p $HOME/miniconda
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
conda init bash
|
|
source ~/.bashrc
|
|
conda create -n myenv python=3.10 -y
|
|
conda activate myenv
|
|
python --version
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source "$HOME/miniconda/etc/profile.d/conda.sh"
|
|
conda activate myenv
|
|
fi
|
|
uv sync --frozen --all-groups --all-extras --python "$(which python)"
|
|
- run:
|
|
name: Install dockerize
|
|
command: |
|
|
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start PostgreSQL Database
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=circle_test \
|
|
-p 5432:5432 \
|
|
postgres:14
|
|
- run:
|
|
name: Wait for PostgreSQL to be ready
|
|
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container with test config
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e AWS_REGION_NAME="us-east-1" \
|
|
-e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- run:
|
|
name: Wait for app to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
- run:
|
|
name: Run Claude Agent SDK E2E Tests
|
|
command: |
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
source $HOME/miniconda/etc/profile.d/conda.sh
|
|
conda activate myenv
|
|
export LITELLM_PROXY_URL="http://localhost:4000"
|
|
export LITELLM_API_KEY="sk-1234"
|
|
pwd
|
|
ls
|
|
uv run --no-sync python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
upload-coverage:
|
|
docker:
|
|
- image: cimg/python:3.9
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: .
|
|
# Check file locations
|
|
- run:
|
|
name: Check coverage file location
|
|
command: |
|
|
echo "Current directory:"
|
|
ls -la
|
|
echo "\nContents of tests/llm_translation:"
|
|
ls -la tests/llm_translation
|
|
- run:
|
|
name: Combine Coverage
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
uv run --with 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage
|
|
uv run --with 'coverage[toml]==7.10.6' coverage xml
|
|
- codecov/upload:
|
|
file: ./coverage.xml
|
|
|
|
publish_proxy_extras:
|
|
docker:
|
|
- image: cimg/python:3.12
|
|
working_directory: ~/project/litellm-proxy-extras
|
|
environment:
|
|
TWINE_USERNAME: __token__
|
|
|
|
steps:
|
|
- checkout:
|
|
path: ~/project
|
|
|
|
- run:
|
|
name: Check if litellm-proxy-extras dir or pyproject.toml was modified
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
# Get current version from pyproject.toml
|
|
CURRENT_VERSION=$(python -c 'import tomllib; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); print(data["project"]["version"])')
|
|
|
|
# Get last published version from PyPI
|
|
LAST_VERSION=$(curl -s https://pypi.org/pypi/litellm-proxy-extras/json | python -c "import json, sys; print(json.load(sys.stdin)['info']['version'])")
|
|
|
|
echo "Current version: $CURRENT_VERSION"
|
|
echo "Last published version: $LAST_VERSION"
|
|
|
|
# Compare versions using Python's packaging.version
|
|
VERSION_COMPARE=$(uv run --with 'packaging==25.0' python -c "from packaging import version; print(1 if version.parse('$CURRENT_VERSION') < version.parse('$LAST_VERSION') else 0)")
|
|
|
|
echo "Version compare: $VERSION_COMPARE"
|
|
if [ "$VERSION_COMPARE" = "1" ]; then
|
|
echo "Error: Current version ($CURRENT_VERSION) is less than last published version ($LAST_VERSION)"
|
|
exit 1
|
|
fi
|
|
|
|
# If versions are equal or current is greater, compare against the published package contents.
|
|
EXTRACTED_DIR=$(uv run --with "litellm-proxy-extras==$LAST_VERSION" python -c 'import importlib.util; from pathlib import Path; spec = importlib.util.find_spec("litellm_proxy_extras"); assert spec is not None and spec.origin is not None, "litellm_proxy_extras not found in uv-run environment"; print(Path(spec.origin).resolve().parent)')
|
|
|
|
# Compare contents
|
|
if ! diff -r "$EXTRACTED_DIR" ./litellm_proxy_extras; then
|
|
if [ "$CURRENT_VERSION" = "$LAST_VERSION" ]; then
|
|
echo "Error: Changes detected in litellm-proxy-extras but version was not bumped"
|
|
echo "Current version: $CURRENT_VERSION"
|
|
echo "Last published version: $LAST_VERSION"
|
|
echo "Changes:"
|
|
diff -r "$EXTRACTED_DIR" ./litellm_proxy_extras
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "No changes detected in litellm-proxy-extras. Skipping PyPI publish."
|
|
circleci step halt
|
|
fi
|
|
|
|
- run:
|
|
name: Get new version
|
|
command: |
|
|
NEW_VERSION=$(python -c 'import tomllib; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); print(data["project"]["version"])')
|
|
echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV
|
|
|
|
- run:
|
|
name: Check if versions match
|
|
command: |
|
|
cd ~/project
|
|
# Check pyproject.toml
|
|
CURRENT_VERSION=$(uv run --with 'packaging==25.0' python -c 'import tomllib; from packaging.requirements import Requirement; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); matches = [spec.version for requirement in data["project"]["optional-dependencies"]["proxy"] for parsed in [Requirement(requirement)] if parsed.name == "litellm-proxy-extras" and parsed.specifier for spec in parsed.specifier if spec.operator == "=="]; print(matches[0] if matches else (_ for _ in ()).throw(SystemExit("Could not find exact litellm-proxy-extras pin in project.optional-dependencies.proxy")))')
|
|
if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then
|
|
echo "Error: Version in pyproject.toml ($CURRENT_VERSION) doesn't match new version ($NEW_VERSION)"
|
|
exit 1
|
|
fi
|
|
|
|
- run:
|
|
name: Publish to PyPI
|
|
command: |
|
|
echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
rm -rf build dist
|
|
uv build
|
|
uv tool run --from 'twine==6.2.0' twine upload --verbose dist/*
|
|
|
|
ui_build:
|
|
docker:
|
|
- image: cimg/node:20.19
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
resource_class: medium+
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
- ui-build-deps-v1-
|
|
- restore_cache:
|
|
keys:
|
|
- ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
- ui-nextjs-cache-v1-
|
|
- run:
|
|
name: Install dependencies
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm ci
|
|
- save_cache:
|
|
key: ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
paths:
|
|
- ui/litellm-dashboard/node_modules
|
|
- run:
|
|
name: Build UI
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
source ./build_ui.sh
|
|
- save_cache:
|
|
key: ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
paths:
|
|
- ui/litellm-dashboard/.next/cache
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- litellm/proxy/_experimental/out
|
|
|
|
ui_unit_tests:
|
|
docker:
|
|
- image: cimg/node:20.19
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
resource_class: xlarge
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
- ui-unit-deps-v1-
|
|
- run:
|
|
name: Install dependencies
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm ci
|
|
- save_cache:
|
|
key: ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
paths:
|
|
- ui/litellm-dashboard/node_modules
|
|
- run:
|
|
name: Run UI unit tests (Vitest)
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
|
|
CI=true npm run test -- --run \
|
|
--pool forks --poolOptions.forks.maxForks=8
|
|
|
|
e2e_ui_testing:
|
|
docker:
|
|
- image: cimg/python:3.12-browsers
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
- image: cimg/postgres:16.0
|
|
environment:
|
|
POSTGRES_USER: e2euser
|
|
POSTGRES_PASSWORD: e2epassword
|
|
POSTGRES_DB: litellm_e2e
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
environment:
|
|
DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
|
|
CI: "true"
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- ui-e2e-py-deps-v1-{{ checksum "requirements.txt" }}
|
|
- run:
|
|
name: Install Python dependencies
|
|
command: |
|
|
python -m pip install --upgrade pip uv
|
|
uv pip install --system -r requirements.txt
|
|
pip install "prisma==0.11.0"
|
|
prisma generate --schema litellm/proxy/schema.prisma
|
|
- save_cache:
|
|
key: ui-e2e-py-deps-v1-{{ checksum "requirements.txt" }}
|
|
paths:
|
|
- ~/.local/lib
|
|
- ~/.local/bin
|
|
- restore_cache:
|
|
keys:
|
|
- ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
- run:
|
|
name: Install Node dependencies and Playwright
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm ci
|
|
npx playwright install chromium --with-deps
|
|
- save_cache:
|
|
key: ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
paths:
|
|
- ui/litellm-dashboard/node_modules
|
|
- run:
|
|
name: Build UI from source
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm run build
|
|
cp -r out/ ../../litellm/proxy/_experimental/out/
|
|
# Restructure HTML so extensionless routes work (login.html -> login/index.html)
|
|
find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do
|
|
d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html"
|
|
done
|
|
- run:
|
|
name: Wait for PostgreSQL
|
|
command: dockerize -wait tcp://localhost:5432 -timeout 30s
|
|
- run:
|
|
name: Push Prisma schema
|
|
command: prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
|
|
- run:
|
|
name: Seed database
|
|
command: |
|
|
PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
|
|
-f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
|
|
- run:
|
|
name: Start mock LLM server
|
|
command: python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
|
|
background: true
|
|
- run:
|
|
name: Start LiteLLM proxy
|
|
environment:
|
|
LITELLM_MASTER_KEY: "sk-1234"
|
|
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
|
|
DISABLE_SCHEMA_UPDATE: "true"
|
|
SERVER_ROOT_PATH: ""
|
|
PROXY_LOGOUT_URL: ""
|
|
command: |
|
|
python -m litellm.proxy.proxy_cli \
|
|
--config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
|
|
--port 4000
|
|
background: true
|
|
- run:
|
|
name: Wait for proxy to be ready
|
|
command: |
|
|
for i in $(seq 1 60); do
|
|
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true)
|
|
if [ "$HTTP_CODE" = "200" ]; then
|
|
echo "Proxy is ready"
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "Proxy failed to start"
|
|
exit 1
|
|
- run:
|
|
name: Run Playwright E2E tests
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npx playwright test --config e2e_tests/playwright.config.ts
|
|
no_output_timeout: 10m
|
|
- store_artifacts:
|
|
path: ui/litellm-dashboard/test-results
|
|
destination: e2e-test-results
|
|
- store_artifacts:
|
|
path: ui/litellm-dashboard/playwright-report
|
|
destination: e2e-playwright-report
|
|
|
|
build_docker_database_image:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
|
|
- run:
|
|
name: Build Docker image
|
|
command: |
|
|
docker build \
|
|
-t litellm-docker-database:ci \
|
|
-f docker/Dockerfile.database .
|
|
|
|
- run:
|
|
name: Save Docker image to workspace root
|
|
command: |
|
|
docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst
|
|
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- litellm-docker-database.tar.zst
|
|
|
|
|
|
prisma_schema_sync:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: medium
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Install dockerize
|
|
command: |
|
|
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start PostgreSQL Database
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=litellm_schema_sync \
|
|
-p 5432:5432 \
|
|
postgres:14
|
|
- run:
|
|
name: Wait for PostgreSQL to be ready
|
|
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run schema sync via prisma db push
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
--name schema-sync \
|
|
--add-host=host.docker.internal:host-gateway \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--use_prisma_db_push
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f schema-sync
|
|
background: true
|
|
- run:
|
|
name: Wait for proxy to be ready (schema sync complete)
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
- run:
|
|
name: Stop schema sync container
|
|
command: docker stop schema-sync
|
|
|
|
|
|
test_bad_database_url:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: medium
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- setup_google_dns
|
|
- run:
|
|
name: Install dockerize
|
|
command: |
|
|
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start PostgreSQL Database
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=circle_test \
|
|
-p 5432:5432 \
|
|
postgres:14
|
|
- run:
|
|
name: Wait for PostgreSQL to be ready
|
|
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker tag litellm-docker-database:ci myapp:latest
|
|
- run:
|
|
name: Run Docker container with bad DATABASE_URL
|
|
command: |
|
|
docker run --name my-app \
|
|
-p 4000:4000 \
|
|
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
|
|
-e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
|
|
myapp:latest \
|
|
--port 4000 > docker_output.log 2>&1 || true
|
|
- run:
|
|
name: Display Docker logs
|
|
command: cat docker_output.log
|
|
- run:
|
|
name: Check for expected error
|
|
command: |
|
|
if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \
|
|
(grep -q "Database setup failed after multiple retries" docker_output.log || \
|
|
grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then
|
|
echo "Expected error found. Test passed."
|
|
else
|
|
echo "Expected error not found. Test failed."
|
|
cat docker_output.log
|
|
exit 1
|
|
fi
|
|
|
|
workflows:
|
|
version: 2
|
|
build_and_test:
|
|
jobs:
|
|
- using_litellm_on_windows:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- mypy_linting:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- semgrep:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- local_testing_part1:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- local_testing_part2:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- langfuse_logging_unit_tests:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- main
|
|
- /litellm_.*/
|
|
- main
|
|
- /litellm_.*/
|
|
- main
|
|
- /litellm_.*/
|
|
- main
|
|
- /litellm_.*/
|
|
- main
|
|
- /litellm_.*/
|
|
- litellm_assistants_api_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- litellm_router_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- litellm_router_unit_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- check_code_and_doc_quality:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- ui_build:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- ui_unit_tests:
|
|
requires:
|
|
- ui_build
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- auth_ui_unit_tests:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- build_docker_database_image:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- prisma_schema_sync:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- e2e_ui_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- build_and_test:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- e2e_openai_endpoints:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- proxy_logging_guardrails_model_info_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- proxy_spend_accuracy_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- proxy_multi_instance_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- proxy_store_model_in_db_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- proxy_build_from_pip_tests:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- proxy_pass_through_endpoint_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- proxy_e2e_anthropic_messages_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- llm_translation_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- realtime_translation_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- mcp_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- agent_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- guardrails_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- google_generate_content_endpoint_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- llm_responses_api_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- ocr_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- search_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- litellm_mapped_enterprise_tests:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- litellm_mapped_tests_proxy_part1:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- litellm_mapped_tests_proxy_part2:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- main
|
|
- /litellm_.*/
|
|
- main
|
|
- /litellm_.*/
|
|
- main
|
|
- /litellm_.*/
|
|
- main
|
|
- /litellm_.*/
|
|
- main
|
|
- /litellm_.*/
|
|
- batches_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- litellm_utils_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- pass_through_unit_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- image_gen_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- logging_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- audio_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- redis_caching_unit_tests:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- upload-coverage:
|
|
requires:
|
|
- realtime_translation_testing
|
|
- mcp_testing
|
|
- agent_testing
|
|
- google_generate_content_endpoint_testing
|
|
- guardrails_testing
|
|
- ocr_testing
|
|
- search_testing
|
|
- litellm_mapped_tests_proxy_part1
|
|
- litellm_mapped_tests_proxy_part2
|
|
- litellm_mapped_enterprise_tests
|
|
- batches_testing
|
|
- litellm_utils_testing
|
|
- pass_through_unit_testing
|
|
- image_gen_testing
|
|
- logging_testing
|
|
- audio_testing
|
|
- redis_caching_unit_tests
|
|
- langfuse_logging_unit_tests
|
|
- local_testing_part1
|
|
- local_testing_part2
|
|
- litellm_assistants_api_testing
|
|
- db_migration_disable_update_check:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- installing_litellm_on_python:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- installing_litellm_on_python_3_13:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- helm_chart_testing:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- test_bad_database_url:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- publish_proxy_extras:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_release_day_.*/
|