version: 2.1 orbs: codecov: codecov/codecov@4.0.1 node: circleci/node@5.1.0 # Add this line to declare the node orb win: circleci/windows@5.0 # Add Windows orb commands: setup_google_dns: steps: - run: name: "Configure Google DNS" command: | # Backup original resolv.conf sudo cp /etc/resolv.conf /etc/resolv.conf.backup # Add both local and Google DNS servers echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf setup_litellm_enterprise_pip: steps: - run: name: "Install local version of litellm-enterprise" command: | # litellm-enterprise is a uv workspace member and is already installed # by the main `uv sync --all-groups --all-extras`. Do NOT run # `uv sync --package litellm-enterprise` here — that overwrites the # shared .venv and strips out dev/test deps (pytest, prisma, etc.). uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)" setup_litellm_test_deps: steps: - checkout - setup_google_dns - restore_cache: keys: - v3-litellm-uv-deps-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" uv sync --frozen --all-groups --all-extras --python "$(which python)" - setup_litellm_enterprise_pip - save_cache: paths: - ~/.local/lib - ~/.local/bin - ~/.cache/uv key: v3-litellm-uv-deps-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} jobs: # Add Windows testing job using_litellm_on_windows: executor: name: win/default shell: powershell.exe working_directory: ~/project steps: - checkout - run: name: Install Python command: | choco install python --version=3.11.0 -y --no-progress --force refreshenv python --version environment: CHOCOLATEY_CONFIRM_ALL: "true" - run: name: Install Dependencies command: | Invoke-RestMethod https://astral.sh/uv/0.10.9/install.ps1 | Invoke-Expression $uvBin = Join-Path $HOME ".local\bin" $env:Path = "$uvBin;$env:Path" if (!(Test-Path $PROFILE)) { New-Item -ItemType File -Force -Path $PROFILE | Out-Null } if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) { Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`"" } uv sync --frozen --group dev --python (Get-Command python).Source - run: name: Run Windows-specific test command: | uv run --no-sync python -m pytest tests/windows_tests/test_litellm_on_windows.py -v mypy_linting: docker: - image: cimg/python:3.12 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: medium steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" uv sync --frozen --group dev --python "$(which python)" --no-install-package fastuuid - run: name: MyPy Type Checking command: | cd litellm # Use the same approach as GitHub Actions, explicitly exclude fastuuid to avoid segfaults uv run --no-sync python -m mypy . cd .. no_output_timeout: 10m semgrep: docker: - image: cimg/python:3.12 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: medium steps: - checkout - setup_google_dns - run: name: Install Semgrep command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" - run: name: Run Semgrep (custom rules only) command: | echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" uv tool run --from 'semgrep==1.157.0' semgrep scan --config .semgrep/rules . --error local_testing_part1: docker: - image: cimg/python:3.12 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project parallelism: 4 steps: - checkout - setup_google_dns - run: name: Show git commit hash command: | echo "Git commit hash: $CIRCLE_SHA1" - restore_cache: keys: - v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - setup_litellm_enterprise_pip - save_cache: paths: - ./.venv key: v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Run prisma ./docker/entrypoint.sh command: | set +e chmod +x docker/entrypoint.sh ./docker/entrypoint.sh set -e - run: name: Black Formatting command: | cd litellm uv run --no-sync python -m black . cd .. # Run pytest and generate JUnit XML report - run: name: Run tests (Part 1 - A-M) command: | mkdir test-results # Discover test files (A-M) TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py") echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ --command="xargs uv run --no-sync python -m pytest \ -vv \ --cov=litellm \ --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=20 \ -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ -n 4 \ --timeout=300 \ --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml local_testing_part1_coverage.xml mv .coverage local_testing_part1_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - local_testing_part1_coverage.xml - local_testing_part1_coverage local_testing_part2: docker: - image: cimg/python:3.12 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project parallelism: 4 steps: - checkout - setup_google_dns - run: name: Show git commit hash command: | echo "Git commit hash: $CIRCLE_SHA1" - restore_cache: keys: - v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - setup_litellm_enterprise_pip - save_cache: paths: - ./.venv key: v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Run prisma ./docker/entrypoint.sh command: | set +e chmod +x docker/entrypoint.sh ./docker/entrypoint.sh set -e - run: name: Black Formatting command: | cd litellm uv run --no-sync python -m black . cd .. # Run pytest and generate JUnit XML report - run: name: Run tests (Part 2 - N-Z) command: | mkdir test-results # Discover test files (N-Z) TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py") echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ --command="xargs uv run --no-sync python -m pytest \ -vv \ --cov=litellm \ --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=20 \ -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ -n 4 \ --timeout=300 \ --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml local_testing_part2_coverage.xml mv .coverage local_testing_part2_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - local_testing_part2_coverage.xml - local_testing_part2_coverage langfuse_logging_unit_tests: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: medium steps: - checkout - setup_google_dns - run: name: Show git commit hash command: | echo "Git commit hash: $CIRCLE_SHA1" - restore_cache: keys: - v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - setup_litellm_enterprise_pip - save_cache: paths: - ./.venv key: v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Run prisma ./docker/entrypoint.sh command: | set +e chmod +x docker/entrypoint.sh ./docker/entrypoint.sh set -e # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse" no_output_timeout: 15m # Store test results - store_test_results: path: test-results auth_ui_unit_tests: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - save_cache: paths: - ./.venv key: v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Run prisma ./docker/entrypoint.sh command: | set +e chmod +x docker/entrypoint.sh ./docker/entrypoint.sh set -e - run: name: Generate Prisma Client command: uv run --no-sync python -m prisma generate # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2 no_output_timeout: 15m # Store test results - store_test_results: path: test-results litellm_router_testing: # Runs all tests with the "router" keyword docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: large parallelism: 4 steps: - checkout - setup_google_dns - restore_cache: keys: - v1-router-testing-deps-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - save_cache: paths: - /home/circleci/.pyenv - /home/circleci/.local key: v1-router-testing-deps-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | pwd ls TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ --command="xargs uv run --no-sync python -m pytest \ -v \ -k 'router' \ -n 4 \ --junitxml=test-results/junit.xml \ --durations=5 \ --timeout=300 --timeout_method=thread" no_output_timeout: 15m # Store test results - store_test_results: path: test-results litellm_router_unit_testing: # Runs all tests with the "router" keyword docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: large steps: - checkout - setup_google_dns - restore_cache: keys: - v1-router-unit-deps-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - save_cache: paths: - /home/circleci/.pyenv - /home/circleci/.local key: v1-router-unit-deps-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4 no_output_timeout: 15m # Store test results - store_test_results: path: test-results litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword docker: - image: cimg/python:3.13.1 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: medium steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results - store_test_results: path: test-results llm_translation_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: large steps: - checkout - setup_google_dns - restore_cache: keys: - v1-llm-translation-deps-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - save_cache: paths: - /home/circleci/.pyenv - /home/circleci/.local key: v1-llm-translation-deps-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls # Add --timeout to kill hanging tests after 120s (2 min) # Add --durations=20 to show 20 slowest tests for debugging # Subdirectories with dedicated jobs (maintain this list as new jobs are added) IGNORE_DIRS=( "tests/llm_translation/realtime" ) IGNORE_ARGS="" for dir in "${IGNORE_DIRS[@]}"; do IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir" done uv run --no-sync python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 no_output_timeout: 15m # Store test results - store_test_results: path: test-results realtime_translation_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run realtime tests command: | pwd ls # Add --timeout to kill hanging tests after 120s (2 min) # Add --durations=20 to show 20 slowest tests for debugging uv run --no-sync python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml realtime_translation_coverage.xml mv .coverage realtime_translation_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - realtime_translation_coverage.xml - realtime_translation_coverage mcp_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -vv tests/mcp_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml mcp_coverage.xml mv .coverage mcp_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - mcp_coverage.xml - mcp_coverage agent_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml agent_coverage.xml mv .coverage agent_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - agent_coverage.xml - agent_coverage guardrails_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml guardrails_coverage.xml mv .coverage guardrails_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - guardrails_coverage.xml - guardrails_coverage google_generate_content_endpoint_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5 no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml google_generate_content_endpoint_coverage.xml mv .coverage google_generate_content_endpoint_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - google_generate_content_endpoint_coverage.xml - google_generate_content_endpoint_coverage llm_responses_api_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: large steps: - checkout - setup_google_dns - restore_cache: keys: - v1-llm-responses-deps-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - save_cache: paths: - /home/circleci/.pyenv - /home/circleci/.local key: v1-llm-responses-deps-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8 no_output_timeout: 15m # Store test results - store_test_results: path: test-results ocr_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml ocr_coverage.xml mv .coverage ocr_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - ocr_coverage.xml - ocr_coverage search_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml search_coverage.xml mv .coverage search_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - search_coverage.xml - search_coverage # Split litellm_mapped_tests into parallel jobs litellm_mapped_tests_proxy_part1: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: large steps: - setup_litellm_test_deps - run: name: Run proxy tests part 1 (high-volume directories) command: | uv run --no-sync python -m prisma generate export PYTHONUNBUFFERED=1 uv run --no-sync python -m pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/client tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part1.xml --durations=10 -n 4 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A no_output_timeout: 15m - store_test_results: path: test-results litellm_mapped_tests_proxy_part2: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: large steps: - setup_litellm_test_deps - run: name: Run proxy tests part 2 (all other tests) command: | uv run --no-sync python -m prisma generate export PYTHONUNBUFFERED=1 uv run --no-sync python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A no_output_timeout: 15m - store_test_results: path: test-results litellm_mapped_enterprise_tests: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: large steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - setup_litellm_enterprise_pip - run: name: Run enterprise tests command: | pwd ls uv run --no-sync python -m prisma generate uv run --no-sync python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4 no_output_timeout: 15m # Store test results - store_test_results: path: test-results batches_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml batches_coverage.xml mv .coverage batches_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - batches_coverage.xml - batches_coverage litellm_utils_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml litellm_utils_coverage.xml mv .coverage litellm_utils_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - litellm_utils_coverage.xml - litellm_utils_coverage pass_through_unit_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml pass_through_unit_tests_coverage.xml mv .coverage pass_through_unit_tests_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - pass_through_unit_tests_coverage.xml - pass_through_unit_tests_coverage image_gen_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: large steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results - store_test_results: path: test-results logging_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | pwd ls LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml logging_coverage.xml || true mv .coverage logging_coverage || true # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - logging_coverage.xml - logging_coverage audio_testing: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml audio_coverage.xml mv .coverage audio_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - audio_coverage.xml - audio_coverage redis_caching_unit_tests: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - restore_cache: keys: - v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" uv sync --frozen --all-groups --all-extras --python "$(which python)" - save_cache: paths: - ./.venv key: v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | uv run --no-sync python -m pytest -vv \ tests/local_testing/test_dual_cache.py \ tests/local_testing/test_redis_batch_optimizations.py \ tests/local_testing/test_router_utils.py \ --cov=litellm --cov-report=xml \ -x -s -v --junitxml=test-results/junit.xml \ --durations=5 -n 2 \ --reruns 2 --reruns-delay 1 no_output_timeout: 20m - run: name: Rename the coverage files command: | mv coverage.xml redis_caching_coverage.xml mv .coverage redis_caching_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - redis_caching_coverage.xml - redis_caching_coverage installing_litellm_on_python: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - setup_litellm_enterprise_pip - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py installing_litellm_on_python_3_13: docker: - image: cimg/python:3.13.1 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: medium steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py helm_chart_testing: machine: image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker resource_class: medium working_directory: ~/project steps: - checkout - attach_workspace: at: ~/project - setup_google_dns # Install Helm - run: name: Install Helm command: | curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash # Install kind - run: name: Install Kind command: | curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 chmod +x ./kind sudo mv ./kind /usr/local/bin/kind # Install kubectl - run: name: Install kubectl command: | curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" chmod +x kubectl sudo mv kubectl /usr/local/bin/ # Create kind cluster - run: name: Create Kind Cluster command: | kind create cluster --name litellm-test - run: name: Load Docker Database Image for helm tests command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load IMAGE_TAG=${CIRCLE_SHA1:-ci} docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG} - run: name: Load Docker image into Kind command: | IMAGE_TAG=${CIRCLE_SHA1:-ci} kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test # Run helm lint - run: name: Run helm lint command: | helm lint ./deploy/charts/litellm-helm # Run helm tests - run: name: Run helm tests command: | IMAGE_TAG=${CIRCLE_SHA1:-ci} helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \ --set image.repository=litellm-ci \ --set image.tag=${IMAGE_TAG} \ --set image.pullPolicy=Never # Wait for pod to be ready echo "Waiting 30 seconds for pod to be ready..." sleep 30 # Print pod logs before running tests echo "Printing pod logs..." kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}") # Run the helm tests helm test litellm --logs helm test litellm --logs # Cleanup - run: name: Cleanup command: | kind delete cluster --name litellm-test when: always # This ensures cleanup runs even if previous steps fail check_code_and_doc_quality: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project/litellm steps: - checkout - setup_google_dns - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) - run: uv run --no-sync ruff check ./litellm # - run: python ./tests/documentation_tests/test_general_setting_keys.py - run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py - run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py - run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py - run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py - run: uv run --no-sync python ./tests/code_coverage_tests/info_log_check.py - run: uv run --no-sync python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py - run: uv run --no-sync python ./tests/code_coverage_tests/test_ban_set_verbose.py - run: uv run --no-sync python ./tests/code_coverage_tests/code_qa_check_tests.py - run: uv run --no-sync python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py - run: uv run --no-sync python ./tests/code_coverage_tests/test_proxy_types_import.py - run: uv run --no-sync python ./tests/code_coverage_tests/callback_manager_test.py - run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py - run: uv run --no-sync python ./tests/code_coverage_tests/test_router_strategy_async.py - run: uv run --no-sync python ./tests/code_coverage_tests/litellm_logging_code_coverage.py - run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py - run: uv run --no-sync python ./tests/documentation_tests/test_router_settings.py - run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py - run: uv run --no-sync python ./tests/code_coverage_tests/ensure_async_clients_test.py - run: uv run --no-sync python ./tests/code_coverage_tests/enforce_llms_folder_style.py - run: uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py - run: uv run --no-sync python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py - run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py - run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py - run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py - run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py - run: helm lint ./deploy/charts/litellm-helm db_migration_disable_update_check: machine: image: ubuntu-2204:2023.10.1 resource_class: medium working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Python 3.9 command: | curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" conda init bash source ~/.bashrc conda create -n myenv python=3.9 -y conda activate myenv python --version - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Install dockerize command: | sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start PostgreSQL Database command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=litellm_test \ -p 5432:5432 \ postgres:14 - run: name: Wait for PostgreSQL to be ready command: dockerize -wait tcp://localhost:5432 -timeout 1m - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Seed database with real schema command: | docker run -d \ -p 4001:4000 \ -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ -e LITELLM_MASTER_KEY="sk-1234" \ --name schema-seed \ --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --use_prisma_db_push - run: name: Wait for schema seed to complete command: dockerize -wait http://localhost:4001 -timeout 5m - run: name: Stop schema seed container command: docker stop schema-seed && docker rm schema-seed - run: name: Run Docker container with bad schema and disabled updates command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DISABLE_SCHEMA_UPDATE="True" \ --name my-app \ --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - run: name: Wait for container to be ready command: dockerize -wait http://localhost:4000 -timeout 1m - run: name: Check container logs for expected message command: | echo "=== Printing Full Container Startup Logs ===" LOG_OUTPUT="$(docker logs my-app 2>&1)" printf '%s\n' "$LOG_OUTPUT" echo "=== End of Full Container Startup Logs ===" if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then echo "Expected message found in logs. Test passed." else echo "Expected message not found in logs. Test failed." exit 1 fi - run: name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) command: | uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 no_output_timeout: 15m build_and_test: machine: image: ubuntu-2204:2023.10.1 resource_class: large working_directory: ~/project steps: - checkout - attach_workspace: at: ~/project - setup_google_dns - run: name: Install Python 3.9 command: | curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" conda init bash source ~/.bashrc conda create -n myenv python=3.9 -y conda activate myenv python --version - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Install dockerize command: | wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start PostgreSQL Database command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - run: name: Wait for PostgreSQL to be ready command: dockerize -wait tcp://localhost:5432 -timeout 1m - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker tag litellm-docker-database:ci my-app:latest - run: name: Run Docker container command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e USE_PRISMA_MIGRATE=True \ -e AZURE_API_KEY=$AZURE_API_KEY \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e GROQ_API_KEY=$GROQ_API_KEY \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e AUTO_INFER_REGION=True \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ my-app:latest \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Install curl command: | sudo apt-get update sudo apt-get install -y curl - run: name: Start outputting logs command: docker logs -f my-app background: true - run: name: Wait for app to be ready command: dockerize -wait http://localhost:4000 -timeout 5m - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests no_output_timeout: 15m # Store test results - store_test_results: path: test-results e2e_openai_endpoints: machine: image: ubuntu-2204:2023.10.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Docker CLI (In case it's not already installed) command: | curl -fsSL https://get.docker.com | sh sudo usermod -aG docker $USER docker version - run: name: Install Python 3.10 command: | curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" conda init bash source ~/.bashrc conda create -n myenv python=3.10 -y conda activate myenv python --version - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Install dockerize command: | wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start PostgreSQL Database command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - run: name: Wait for PostgreSQL to be ready command: dockerize -wait tcp://localhost:5432 -timeout 1m - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e AZURE_API_KEY=$AZURE_API_KEY \ -e AZURE_API_BASE=$AZURE_API_BASE \ -e AZURE_API_VERSION="2024-05-01-preview" \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e GROQ_API_KEY=$GROQ_API_KEY \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e AUTO_INFER_REGION=True \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Install curl command: | sudo apt-get update sudo apt-get install -y curl - run: name: Start outputting logs command: docker logs -f my-app background: true - run: name: Wait for app to be ready command: dockerize -wait http://localhost:4000 -timeout 5m - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results - store_test_results: path: test-results proxy_logging_guardrails_model_info_tests: machine: image: ubuntu-2204:2023.10.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Docker CLI (In case it's not already installed) command: | curl -fsSL https://get.docker.com | sh sudo usermod -aG docker $USER docker version - run: name: Install Python 3.9 command: | curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" conda init bash source ~/.bashrc conda create -n myenv python=3.9 -y conda activate myenv python --version - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Install dockerize command: | wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start PostgreSQL Database command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - run: name: Wait for PostgreSQL to be ready command: dockerize -wait tcp://localhost:5432 -timeout 1m - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container # intentionally give bad redis credentials here # the OTEL test - should get this as a trace command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e GCS_FLUSH_INTERVAL="1" \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \ -v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Install curl and dockerize command: | sudo apt-get update sudo apt-get install -y curl sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app background: true - run: name: Wait for app to be ready command: dockerize -wait http://localhost:4000 -timeout 5m - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Clean up first container - run: name: Stop and remove first container command: | docker stop my-app docker rm my-app # Second Docker Container Run with Different Config # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license - run: name: Run Second Docker container command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE="bad-license" \ --add-host host.docker.internal:host-gateway \ --name my-app-3 \ -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug - run: name: Start outputting logs for second container command: docker logs -f my-app-3 background: true - run: name: Wait for second app to be ready command: dockerize -wait http://localhost:4000 -timeout 5m - run: name: Run second round of tests command: | uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 no_output_timeout: 15m # Store test results - store_test_results: path: test-results proxy_spend_accuracy_tests: machine: image: ubuntu-2204:2023.10.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Docker CLI (In case it's not already installed) command: | curl -fsSL https://get.docker.com | sh sudo usermod -aG docker $USER docker version - run: name: Install Python 3.9 command: | curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" conda init bash source ~/.bashrc conda create -n myenv python=3.9 -y conda activate myenv python --version - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Install dockerize command: | wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start PostgreSQL Database command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - run: name: Wait for PostgreSQL to be ready command: dockerize -wait tcp://localhost:5432 -timeout 1m - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container # intentionally give bad redis credentials here # the OTEL test - should get this as a trace command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Install curl command: | sudo apt-get update sudo apt-get install -y curl - run: name: Start outputting logs command: docker logs -f my-app background: true - run: name: Wait for app to be ready command: dockerize -wait http://localhost:4000 -timeout 5m - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Clean up first container - run: name: Stop and remove first container command: | docker stop my-app docker rm my-app proxy_multi_instance_tests: machine: image: ubuntu-2204:2023.10.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Docker CLI (In case it's not already installed) command: | curl -fsSL https://get.docker.com | sh sudo usermod -aG docker $USER docker version - run: name: Install Python 3.9 command: | curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" conda init bash source ~/.bashrc conda create -n myenv python=3.9 -y conda activate myenv python --version - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Install dockerize command: | wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start PostgreSQL Database command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - run: name: Wait for PostgreSQL to be ready command: dockerize -wait tcp://localhost:5432 -timeout 1m - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container 1 # intentionally give bad redis credentials here # the OTEL test - should get this as a trace command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Run Docker container 2 command: | docker run -d \ -p 4001:4001 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ --add-host host.docker.internal:host-gateway \ --name my-app-2 \ -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4001 \ --detailed_debug - run: name: Install curl and dockerize command: | sudo apt-get update sudo apt-get install -y curl sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app background: true - run: name: Wait for instance 1 to be ready command: dockerize -wait http://localhost:4000 -timeout 5m - run: name: Wait for instance 2 to be ready command: dockerize -wait http://localhost:4001 -timeout 5m - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Clean up first container # Store test results - store_test_results: path: test-results proxy_store_model_in_db_tests: machine: image: ubuntu-2204:2023.10.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Docker CLI (In case it's not already installed) command: | curl -fsSL https://get.docker.com | sh sudo usermod -aG docker $USER docker version sudo systemctl restart docker - run: name: Install Python 3.9 command: | curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" conda init bash source ~/.bashrc conda create -n myenv python=3.9 -y conda activate myenv python --version - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Install dockerize command: | wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start PostgreSQL Database command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - run: name: Wait for PostgreSQL to be ready command: dockerize -wait tcp://localhost:5432 -timeout 1m - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container # intentionally give bad redis credentials here # the OTEL test - should get this as a trace command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e STORE_MODEL_IN_DB="True" \ -e LITELLM_MASTER_KEY="sk-1234" \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Install curl and dockerize command: | sudo apt-get update sudo apt-get install -y curl sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app background: true - run: name: Wait for app to be ready command: dockerize -wait http://localhost:4000 -timeout 5m - run: name: Run tests command: | pwd ls uv run --no-sync python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m - run: name: Stop and remove containers command: | docker stop my-app || true docker rm my-app || true docker stop postgres-db || true docker rm postgres-db || true when: always - store_test_results: path: test-results proxy_build_from_pip_tests: # Change from docker to machine executor machine: image: ubuntu-2204:2023.10.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns # Remove Docker CLI installation since it's already available in machine executor - run: name: Install Python 3.13 command: | curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" conda init bash source ~/.bashrc conda create -n myenv python=3.13 -y conda activate myenv python --version - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Build Docker image command: | docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip . - run: name: Start PostgreSQL Database command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - run: name: Wait for PostgreSQL to be ready command: | timeout 60s bash -c 'until docker exec postgres-db pg_isready -U postgres -d circle_test; do sleep 2; done' - run: name: Run Docker container # intentionally give bad redis credentials here # the OTEL test - should get this as a trace command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e GCS_FLUSH_INTERVAL="1" \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \ my-app:latest \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Install curl and dockerize command: | sudo apt-get update sudo apt-get install -y curl sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app background: true - run: name: Wait for app to be ready command: dockerize -wait http://localhost:4000 -timeout 5m - run: name: Run tests command: | uv run --no-sync python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 no_output_timeout: 15m # Clean up first container - run: name: Stop and remove first container command: | docker stop my-app || true docker rm my-app || true docker stop postgres-db || true docker rm postgres-db || true when: always proxy_pass_through_endpoint_tests: machine: image: ubuntu-2204:2023.10.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Python 3.10 command: | curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" conda init bash source ~/.bashrc conda create -n myenv python=3.10 -y conda activate myenv python --version - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Install dockerize command: | wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start PostgreSQL Database command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - run: name: Wait for PostgreSQL to be ready command: dockerize -wait tcp://localhost:5432 -timeout 1m - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e GEMINI_API_KEY=$GEMINI_API_KEY \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \ -e AZURE_API_KEY=$AZURE_API_KEY \ -e AZURE_API_BASE=$AZURE_API_BASE \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \ -v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Start outputting logs command: docker logs -f my-app background: true - run: name: Wait for app to be ready command: dockerize -wait http://localhost:4000 -timeout 5m # Add Ruby installation and testing before the existing Node.js and Python tests - run: name: Install Ruby and Bundler command: | # Import GPG keys first gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB || { curl -sSL https://rvm.io/mpapis.asc | gpg --import - curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - } # Install Ruby version manager (RVM) curl -sSL https://get.rvm.io | bash -s stable # Source RVM from the correct location source $HOME/.rvm/scripts/rvm # Install Ruby 3.2.2 rvm install 3.2.2 rvm use 3.2.2 --default # Install latest Bundler gem install bundler - run: name: Run Ruby tests command: | source $HOME/.rvm/scripts/rvm cd tests/pass_through_tests/ruby_passthrough_tests bundle install bundle exec rspec no_output_timeout: 30m # New steps to run Node.js test - run: name: Install Node.js command: | export DEBIAN_FRONTEND=noninteractive curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash - sudo apt-get update sudo apt-get install -y nodejs node --version npm --version - run: name: Install Node.js dependencies command: | npm install @google-cloud/vertexai npm install @google/generative-ai npm install --save-dev jest - run: name: Run Vertex AI, Google AI Studio Node.js tests command: | npx jest tests/pass_through_tests --verbose no_output_timeout: 30m - run: name: Run tests command: | export PATH="$HOME/miniconda/bin:$PATH" source $HOME/miniconda/etc/profile.d/conda.sh conda activate myenv pwd ls uv run --no-sync python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results - store_test_results: path: test-results proxy_e2e_anthropic_messages_tests: machine: image: ubuntu-2204:2023.10.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - run: name: Install Docker CLI (In case it's not already installed) command: | curl -fsSL https://get.docker.com | sh sudo usermod -aG docker $USER docker version - run: name: Install Python 3.10 command: | curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" conda init bash source ~/.bashrc conda create -n myenv python=3.10 -y conda activate myenv python --version - run: name: Install Dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then export PATH="$HOME/miniconda/bin:$PATH" source "$HOME/miniconda/etc/profile.d/conda.sh" conda activate myenv fi uv sync --frozen --all-groups --all-extras --python "$(which python)" - run: name: Install dockerize command: | wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start PostgreSQL Database command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - run: name: Wait for PostgreSQL to be ready command: dockerize -wait tcp://localhost:5432 -timeout 1m - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container with test config command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e LITELLM_MASTER_KEY="sk-1234" \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME="us-east-1" \ -e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug - run: name: Start outputting logs command: docker logs -f my-app background: true - run: name: Wait for app to be ready command: dockerize -wait http://localhost:4000 -timeout 5m - run: name: Run Claude Agent SDK E2E Tests command: | export PATH="$HOME/miniconda/bin:$PATH" source $HOME/miniconda/etc/profile.d/conda.sh conda activate myenv export LITELLM_PROXY_URL="http://localhost:4000" export LITELLM_API_KEY="sk-1234" pwd ls uv run --no-sync python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results - store_test_results: path: test-results upload-coverage: docker: - image: cimg/python:3.9 steps: - checkout - attach_workspace: at: . # Check file locations - run: name: Check coverage file location command: | echo "Current directory:" ls -la echo "\nContents of tests/llm_translation:" ls -la tests/llm_translation - run: name: Combine Coverage command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage uv tool run --from 'coverage[toml]==7.10.6' coverage xml - codecov/upload: file: ./coverage.xml publish_proxy_extras: docker: - image: cimg/python:3.12 working_directory: ~/project/litellm-proxy-extras environment: TWINE_USERNAME: __token__ steps: - checkout: path: ~/project - run: name: Check if litellm-proxy-extras dir or pyproject.toml was modified command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" # Get current version from pyproject.toml CURRENT_VERSION=$(python -c 'import tomllib; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); print(data["project"]["version"])') # Get last published version from PyPI LAST_VERSION=$(curl -s https://pypi.org/pypi/litellm-proxy-extras/json | python -c "import json, sys; print(json.load(sys.stdin)['info']['version'])") echo "Current version: $CURRENT_VERSION" echo "Last published version: $LAST_VERSION" # Compare versions using Python's packaging.version VERSION_COMPARE=$(uv run --with 'packaging==25.0' python -c "from packaging import version; print(1 if version.parse('$CURRENT_VERSION') < version.parse('$LAST_VERSION') else 0)") echo "Version compare: $VERSION_COMPARE" if [ "$VERSION_COMPARE" = "1" ]; then echo "Error: Current version ($CURRENT_VERSION) is less than last published version ($LAST_VERSION)" exit 1 fi # If versions are equal or current is greater, compare against the published package contents. EXTRACTED_DIR=$(uv run --with "litellm-proxy-extras==$LAST_VERSION" python -c 'import importlib.util; from pathlib import Path; spec = importlib.util.find_spec("litellm_proxy_extras"); assert spec is not None and spec.origin is not None, "litellm_proxy_extras not found in uv-run environment"; print(Path(spec.origin).resolve().parent)') # Compare contents if ! diff -r "$EXTRACTED_DIR" ./litellm_proxy_extras; then if [ "$CURRENT_VERSION" = "$LAST_VERSION" ]; then echo "Error: Changes detected in litellm-proxy-extras but version was not bumped" echo "Current version: $CURRENT_VERSION" echo "Last published version: $LAST_VERSION" echo "Changes:" diff -r "$EXTRACTED_DIR" ./litellm_proxy_extras exit 1 fi else echo "No changes detected in litellm-proxy-extras. Skipping PyPI publish." circleci step halt fi - run: name: Get new version command: | NEW_VERSION=$(python -c 'import tomllib; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); print(data["project"]["version"])') echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV - run: name: Check if versions match command: | cd ~/project # Check pyproject.toml CURRENT_VERSION=$(uv run --with 'packaging==25.0' python -c 'import tomllib; from packaging.requirements import Requirement; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); matches = [spec.version for requirement in data["project"]["optional-dependencies"]["proxy"] for parsed in [Requirement(requirement)] if parsed.name == "litellm-proxy-extras" and parsed.specifier for spec in parsed.specifier if spec.operator == "=="]; print(matches[0] if matches else (_ for _ in ()).throw(SystemExit("Could not find exact litellm-proxy-extras pin in project.optional-dependencies.proxy")))') if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then echo "Error: Version in pyproject.toml ($CURRENT_VERSION) doesn't match new version ($NEW_VERSION)" exit 1 fi - run: name: Publish to PyPI command: | echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" rm -rf build dist uv build uv tool run --from 'twine==6.2.0' twine upload --verbose dist/* ui_build: docker: - image: cimg/node:20.19 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} resource_class: medium+ working_directory: ~/project steps: - checkout - setup_google_dns - restore_cache: keys: - ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - ui-build-deps-v1- - restore_cache: keys: - ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - ui-nextjs-cache-v1- - run: name: Install dependencies command: | cd ui/litellm-dashboard npm ci - save_cache: key: ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} paths: - ui/litellm-dashboard/node_modules - run: name: Build UI command: | cd ui/litellm-dashboard source ./build_ui.sh - save_cache: key: ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} paths: - ui/litellm-dashboard/.next/cache - persist_to_workspace: root: . paths: - litellm/proxy/_experimental/out ui_unit_tests: docker: - image: cimg/node:20.19 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} resource_class: xlarge working_directory: ~/project steps: - checkout - setup_google_dns - restore_cache: keys: - ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - ui-unit-deps-v1- - run: name: Install dependencies command: | cd ui/litellm-dashboard npm ci - save_cache: key: ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} paths: - ui/litellm-dashboard/node_modules - run: name: Run UI unit tests (Vitest) command: | cd ui/litellm-dashboard CI=true npm run test -- --run \ --pool forks --poolOptions.forks.maxForks=8 e2e_ui_testing: docker: - image: cimg/python:3.12-browsers auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} - image: cimg/postgres:16.0 environment: POSTGRES_USER: e2euser POSTGRES_PASSWORD: e2epassword POSTGRES_DB: litellm_e2e resource_class: large working_directory: ~/project environment: DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" CI: "true" steps: - checkout - setup_google_dns - restore_cache: keys: - ui-e2e-py-deps-v2-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} - run: name: Install Python dependencies command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" uv sync --frozen --all-groups --all-extras --python "$(which python)" uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - save_cache: key: ui-e2e-py-deps-v2-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }} paths: - ./.venv - restore_cache: keys: - ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - run: name: Install Node dependencies and Playwright command: | cd ui/litellm-dashboard npm ci npx playwright install chromium --with-deps - save_cache: key: ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} paths: - ui/litellm-dashboard/node_modules - run: name: Build UI from source command: | cd ui/litellm-dashboard npm run build cp -r out/ ../../litellm/proxy/_experimental/out/ # Restructure HTML so extensionless routes work (login.html -> login/index.html) find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" done - run: name: Wait for PostgreSQL command: dockerize -wait tcp://localhost:5432 -timeout 30s - run: name: Push Prisma schema command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - run: name: Seed database command: | PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql - run: name: Start mock LLM server command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py background: true - run: name: Start LiteLLM proxy environment: LITELLM_MASTER_KEY: "sk-1234" MOCK_LLM_URL: "http://127.0.0.1:8090/v1" DISABLE_SCHEMA_UPDATE: "true" SERVER_ROOT_PATH: "" PROXY_LOGOUT_URL: "" command: | uv run --no-sync python -m litellm.proxy.proxy_cli \ --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \ --port 4000 background: true - run: name: Wait for proxy to be ready command: | for i in $(seq 1 60); do HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true) if [ "$HTTP_CODE" = "200" ]; then echo "Proxy is ready" exit 0 fi sleep 2 done echo "Proxy failed to start" exit 1 - run: name: Run Playwright E2E tests command: | cd ui/litellm-dashboard npx playwright test --config e2e_tests/playwright.config.ts no_output_timeout: 10m - store_artifacts: path: ui/litellm-dashboard/test-results destination: e2e-test-results - store_artifacts: path: ui/litellm-dashboard/playwright-report destination: e2e-playwright-report build_docker_database_image: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - run: name: Build Docker image command: | docker build \ -t litellm-docker-database:ci \ -f docker/Dockerfile.database . - run: name: Save Docker image to workspace root command: | docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst - persist_to_workspace: root: . paths: - litellm-docker-database.tar.zst prisma_schema_sync: machine: image: ubuntu-2204:2023.10.1 resource_class: medium working_directory: ~/project steps: - checkout - setup_google_dns - attach_workspace: at: ~/project - run: name: Install dockerize command: | sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start PostgreSQL Database command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=litellm_schema_sync \ -p 5432:5432 \ postgres:14 - run: name: Wait for PostgreSQL to be ready command: dockerize -wait tcp://localhost:5432 -timeout 1m - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run schema sync via prisma db push command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \ -e LITELLM_MASTER_KEY="sk-1234" \ --name schema-sync \ --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --use_prisma_db_push - run: name: Start outputting logs command: docker logs -f schema-sync background: true - run: name: Wait for proxy to be ready (schema sync complete) command: dockerize -wait http://localhost:4000 -timeout 5m - run: name: Stop schema sync container command: docker stop schema-sync test_bad_database_url: machine: image: ubuntu-2204:2023.10.1 resource_class: medium working_directory: ~/project steps: - checkout - attach_workspace: at: ~/project - setup_google_dns - run: name: Install dockerize command: | wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start PostgreSQL Database command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=circle_test \ -p 5432:5432 \ postgres:14 - run: name: Wait for PostgreSQL to be ready command: dockerize -wait tcp://localhost:5432 -timeout 1m - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker tag litellm-docker-database:ci myapp:latest - run: name: Run Docker container with bad DATABASE_URL command: | docker run --name my-app \ -p 4000:4000 \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \ myapp:latest \ --port 4000 > docker_output.log 2>&1 || true - run: name: Display Docker logs command: cat docker_output.log - run: name: Check for expected error command: | if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \ (grep -q "Database setup failed after multiple retries" docker_output.log || \ grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then echo "Expected error found. Test passed." else echo "Expected error not found. Test failed." cat docker_output.log exit 1 fi workflows: version: 2 build_and_test: jobs: - using_litellm_on_windows: filters: branches: only: - main - /litellm_.*/ - mypy_linting: filters: branches: only: - main - /litellm_.*/ - semgrep: filters: branches: only: - main - /litellm_.*/ - local_testing_part1: filters: branches: only: - main - /litellm_.*/ - local_testing_part2: filters: branches: only: - main - /litellm_.*/ - langfuse_logging_unit_tests: filters: branches: only: - main - /litellm_.*/ - main - /litellm_.*/ - main - /litellm_.*/ - main - /litellm_.*/ - main - /litellm_.*/ - main - /litellm_.*/ - litellm_assistants_api_testing: filters: branches: only: - main - /litellm_.*/ - litellm_router_testing: filters: branches: only: - main - /litellm_.*/ - litellm_router_unit_testing: filters: branches: only: - main - /litellm_.*/ - check_code_and_doc_quality: filters: branches: only: - main - /litellm_.*/ - ui_build: filters: branches: only: - main - /litellm_.*/ - ui_unit_tests: requires: - ui_build filters: branches: only: - main - /litellm_.*/ - auth_ui_unit_tests: filters: branches: only: - main - /litellm_.*/ - build_docker_database_image: filters: branches: only: - main - /litellm_.*/ - prisma_schema_sync: requires: - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - e2e_ui_testing: filters: branches: only: - main - /litellm_.*/ - build_and_test: requires: - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - e2e_openai_endpoints: requires: - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - proxy_logging_guardrails_model_info_tests: requires: - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - proxy_spend_accuracy_tests: requires: - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - proxy_multi_instance_tests: requires: - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - proxy_store_model_in_db_tests: requires: - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - proxy_build_from_pip_tests: filters: branches: only: - main - /litellm_.*/ - proxy_pass_through_endpoint_tests: requires: - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - proxy_e2e_anthropic_messages_tests: requires: - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - llm_translation_testing: filters: branches: only: - main - /litellm_.*/ - realtime_translation_testing: filters: branches: only: - main - /litellm_.*/ - mcp_testing: filters: branches: only: - main - /litellm_.*/ - agent_testing: filters: branches: only: - main - /litellm_.*/ - guardrails_testing: filters: branches: only: - main - /litellm_.*/ - google_generate_content_endpoint_testing: filters: branches: only: - main - /litellm_.*/ - llm_responses_api_testing: filters: branches: only: - main - /litellm_.*/ - ocr_testing: filters: branches: only: - main - /litellm_.*/ - search_testing: filters: branches: only: - main - /litellm_.*/ - litellm_mapped_enterprise_tests: filters: branches: only: - main - /litellm_.*/ - litellm_mapped_tests_proxy_part1: filters: branches: only: - main - /litellm_.*/ - litellm_mapped_tests_proxy_part2: filters: branches: only: - main - /litellm_.*/ - main - /litellm_.*/ - main - /litellm_.*/ - main - /litellm_.*/ - main - /litellm_.*/ - main - /litellm_.*/ - batches_testing: filters: branches: only: - main - /litellm_.*/ - litellm_utils_testing: filters: branches: only: - main - /litellm_.*/ - pass_through_unit_testing: filters: branches: only: - main - /litellm_.*/ - image_gen_testing: filters: branches: only: - main - /litellm_.*/ - logging_testing: filters: branches: only: - main - /litellm_.*/ - audio_testing: filters: branches: only: - main - /litellm_.*/ - redis_caching_unit_tests: filters: branches: only: - main - /litellm_.*/ - upload-coverage: requires: - realtime_translation_testing - mcp_testing - agent_testing - google_generate_content_endpoint_testing - guardrails_testing - ocr_testing - search_testing - litellm_mapped_tests_proxy_part1 - litellm_mapped_tests_proxy_part2 - litellm_mapped_enterprise_tests - batches_testing - litellm_utils_testing - pass_through_unit_testing - image_gen_testing - logging_testing - audio_testing - redis_caching_unit_tests - langfuse_logging_unit_tests - local_testing_part1 - local_testing_part2 - litellm_assistants_api_testing - db_migration_disable_update_check: requires: - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - installing_litellm_on_python: filters: branches: only: - main - /litellm_.*/ - installing_litellm_on_python_3_13: filters: branches: only: - main - /litellm_.*/ - helm_chart_testing: requires: - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - test_bad_database_url: requires: - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - publish_proxy_extras: filters: branches: only: - main - /litellm_release_day_.*/