From f118ecc1b60e9c25ab0e0d5221f830e2728c89b5 Mon Sep 17 00:00:00 2001 From: Chesars Date: Sat, 25 Apr 2026 15:09:00 -0300 Subject: [PATCH] ci: align CI/workflows with litellm_internal_staging Sync CI configs with upstream/litellm_internal_staging: - Migrate from Poetry to uv (PR #25007) - Pull in zizmor security fixes for workflows - Add isolated unit test workflows + reusable bases - Drop redundant matrix workflow and azure-batches workflow - Drop .circleci/requirements.txt (replaced by uv lockfile) --- .circleci/config.yml | 2689 ++++------------- .circleci/requirements.txt | 21 - .github/workflows/_test-unit-base.yml | 77 +- .../workflows/_test-unit-services-base.yml | 137 +- .../auto_update_price_and_context_window.yml | 9 +- .github/workflows/create-release.yml | 27 +- .github/workflows/llm-translation-testing.yml | 25 +- .github/workflows/publish_to_pypi.yml | 89 +- .github/workflows/scan_duplicate_issues.yml | 2 +- .github/workflows/test-linting.yml | 41 +- .github/workflows/test-litellm-matrix.yml | 214 -- .github/workflows/test-litellm.yml | 22 +- .github/workflows/test-mcp.yml | 30 +- .github/workflows/test-model-map.yaml | 6 +- .../test-proxy-e2e-azure-batches.yml | 97 - .github/workflows/test-unit-core-utils.yml | 9 +- .github/workflows/test-unit-documentation.yml | 48 +- .../test-unit-enterprise-routing.yml | 9 +- .github/workflows/test-unit-integrations.yml | 9 +- .github/workflows/test-unit-llm-providers.yml | 16 +- .github/workflows/test-unit-misc.yml | 9 +- .github/workflows/test-unit-proxy-auth.yml | 9 +- .github/workflows/test-unit-proxy-db.yml | 223 +- .../workflows/test-unit-proxy-endpoints.yml | 11 +- .github/workflows/test-unit-proxy-infra.yml | 9 +- .github/workflows/test-unit-proxy-legacy.yml | 37 +- .../test-unit-responses-caching-types.yml | 9 +- .github/workflows/test-unit-security.yml | 14 +- .github/workflows/test_server_root_path.yml | 14 +- scripts/install.sh | 50 +- 30 files changed, 1329 insertions(+), 2633 deletions(-) delete mode 100644 .circleci/requirements.txt delete mode 100644 .github/workflows/test-litellm-matrix.yml delete mode 100644 .github/workflows/test-proxy-e2e-azure-batches.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 307247651e..9f01bae3e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,40 +16,128 @@ commands: echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf + wait_for_service: + description: "Poll a TCP or HTTP endpoint until it responds (replaces dockerize -wait)" + parameters: + url: + type: string + timeout: + type: string + default: "60" + steps: + - run: + name: "Wait for << parameters.url >>" + command: | + TIMEOUT=<< parameters.timeout >> + URL="<< parameters.url >>" + ELAPSED=0 + echo "Waiting up to ${TIMEOUT}s for ${URL} ..." + if echo "$URL" | grep -q '^tcp://'; then + HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1) + PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2) + while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do + sleep 2; ELAPSED=$((ELAPSED+2)) + if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi + done + else + while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do + sleep 2; ELAPSED=$((ELAPSED+2)) + if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi + done + fi + echo "Service ready after ${ELAPSED}s" + install_helm: + steps: + - run: + name: Install Helm v3.17.3 + command: | + curl -sSLf -o /tmp/helm.tar.gz \ + https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz + echo "ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd /tmp/helm.tar.gz" | sha256sum -c - + sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/helm.tar.gz linux-amd64/helm + rm -f /tmp/helm.tar.gz + install_kind: + steps: + - run: + name: Install Kind v0.20.0 + command: | + curl -sSLf -o /tmp/kind \ + https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 + echo "513a7213d6d3332dd9ef27c24dab35e5ef10a04fa27274fe1c14d8a246493ded /tmp/kind" | sha256sum -c - + chmod +x /tmp/kind + sudo mv /tmp/kind /usr/local/bin/kind + install_uv: + description: "Install pinned uv (0.10.9) with checksum verification. Adds ~/.local/bin to PATH." + steps: + - run: + name: Install uv (pinned 0.10.9) + command: | + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + start_postgres: + description: "Start a postgres-db container on port 5432 and wait until it accepts connections." + parameters: + db_name: + type: string + default: circle_test + steps: + - run: + name: Start PostgreSQL + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=<< parameters.db_name >> \ + -p 5432:5432 \ + postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26 + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" + start_redis: + description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys." + steps: + - run: + name: Start Redis + command: | + docker run -d \ + --name redis-cache \ + -p 6379:6379 \ + redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6 + - wait_for_service: + url: tcp://localhost:6379 + timeout: "60" setup_litellm_enterprise_pip: steps: - run: name: "Install local version of litellm-enterprise" command: | - pip install --force-reinstall --no-deps -e enterprise/ + # litellm-enterprise is a uv workspace member and is already installed + # by the main `uv sync --all-groups --all-extras`. Do NOT run + # `uv sync --package litellm-enterprise` here — that overwrites the + # shared .venv and strips out dev/test deps (pytest, prisma, etc.). + uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)" setup_litellm_test_deps: steps: - checkout - setup_google_dns + - install_uv - restore_cache: keys: - - v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} + - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - # Use uv for the heavy requirements.txt (10-100x faster than pip) - uv pip install --system -r requirements.txt - # Use pip for test deps (small set, avoids uv strict-resolution - # conflicts with transitive dep pins like openai<2 and pydantic>=2.11.5) - pip install "pytest-mock==3.12.0" "pytest==7.3.1" "pytest-retry==1.6.3" \ - "pytest-asyncio==0.21.1" "respx==0.22.0" "hypercorn==0.17.3" \ - "pydantic==2.12.5" "mcp==1.26.0" "requests-mock>=1.12.1" \ - "responses==0.25.7" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" \ - "pytest-cov==5.0.0" "semantic_router==0.1.10" "fastapi-offline==1.7.3" \ - "a2a" "parameterized>=0.9.0" + uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - save_cache: paths: - - ~/.local/lib - - ~/.local/bin - ~/.cache/uv - key: v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} + key: v1-uv-cache-{{ checksum "uv.lock" }} jobs: # Add Windows testing job @@ -71,63 +159,33 @@ jobs: - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - pip install pytest - pip install . + $installer = Join-Path $env:TEMP "uv-install.ps1" + Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer + $expected = "d43ffff8d28e7d1e7d1831a212465f12b24a43c7f87f386e95e2a5915aee5d7d" + $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower() + if ($actual -ne $expected) { + throw "uv installer hash mismatch: expected $expected got $actual" + } + & $installer + Remove-Item $installer + $uvBin = Join-Path $HOME ".local\bin" + $env:Path = "$uvBin;$env:Path" + if (!(Test-Path $PROFILE)) { + New-Item -ItemType File -Force -Path $PROFILE | Out-Null + } + if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) { + Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`"" + } + uv sync --frozen --group dev --python (Get-Command python).Source - run: name: Run Windows-specific test command: | - python -m pytest tests/windows_tests/test_litellm_on_windows.py -v - - mypy_linting: - docker: - - image: cimg/python:3.12 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: medium - - steps: - - checkout - - setup_google_dns - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip uninstall fastuuid -y - pip install "mypy==1.18.2" - - run: - name: MyPy Type Checking - command: | - cd litellm - # Use the same approach as GitHub Actions, explicitly exclude fastuuid to avoid segfaults - python -m mypy . - cd .. - no_output_timeout: 10m - - semgrep: - docker: - - image: cimg/python:3.12 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: medium - steps: - - checkout - - setup_google_dns - - run: - name: Install Semgrep - command: pip install semgrep - - run: - name: Run Semgrep (custom rules only) - command: semgrep scan --config .semgrep/rules . --error + uv run --no-sync python -m pytest tests/windows_tests/test_litellm_on_windows.py -v local_testing_part1: docker: - - image: cimg/python:3.12 + - &python312_image + image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} @@ -136,38 +194,19 @@ jobs: steps: - checkout - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - restore_cache: keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - v1-uv-cache-{{ checksum "uv.lock" }} + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \ - "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \ - "boto3==1.42.80" langchain lunary==0.2.5 \ - "azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \ - traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \ - "detect_secrets==1.5.0" "respx==0.22.0" fastapi \ - "gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \ - "apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \ - python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \ - "diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \ - "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1" - pip install semantic_router --no-deps - pip install aurelio_sdk --no-deps - pip uninstall posthog -y + uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - save_cache: paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Run prisma ./docker/entrypoint.sh command: | @@ -175,14 +214,6 @@ jobs: chmod +x docker/entrypoint.sh ./docker/entrypoint.sh set -e - - run: - name: Black Formatting - command: | - cd litellm - python -m pip install black - python -m black . - cd .. - # Run pytest and generate JUnit XML report - run: name: Run tests (Part 1 - A-M) @@ -195,7 +226,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ - --command="xargs python -m pytest \ + --command="xargs uv run --no-sync python -m pytest \ -vv \ --cov=litellm \ --cov-report=xml \ @@ -222,47 +253,25 @@ jobs: - local_testing_part1_coverage local_testing_part2: docker: - - image: cimg/python:3.12 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project parallelism: 4 steps: - checkout - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - restore_cache: keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - v1-uv-cache-{{ checksum "uv.lock" }} + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \ - "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \ - "boto3==1.42.80" langchain lunary==0.2.5 \ - "azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \ - traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \ - "detect_secrets==1.5.0" "respx==0.22.0" fastapi \ - "gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \ - "apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \ - python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \ - "diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \ - "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1" - pip install semantic_router --no-deps - pip install aurelio_sdk --no-deps - pip uninstall posthog -y + uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - save_cache: paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Run prisma ./docker/entrypoint.sh command: | @@ -270,14 +279,6 @@ jobs: chmod +x docker/entrypoint.sh ./docker/entrypoint.sh set -e - - run: - name: Black Formatting - command: | - cd litellm - python -m pip install black - python -m black . - cd .. - # Run pytest and generate JUnit XML report - run: name: Run tests (Part 2 - N-Z) @@ -290,7 +291,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ - --command="xargs python -m pytest \ + --command="xargs uv run --no-sync python -m pytest \ -vv \ --cov=litellm \ --cov-report=xml \ @@ -317,75 +318,26 @@ jobs: - local_testing_part2_coverage langfuse_logging_unit_tests: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project resource_class: medium steps: - checkout - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - restore_cache: keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - v1-uv-cache-{{ checksum "uv.lock" }} + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.133.0" - pip install pyarrow - pip install "boto3==1.42.80" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.25.3" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.28.0 - pip install opentelemetry-sdk==1.28.0 - pip install opentelemetry-exporter-otlp==1.28.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.28.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==23.0.0" - pip install "anyio==4.8.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.11.2" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.12.5" - pip install "diskcache==5.6.1" - pip install "Pillow==12.1.1" - pip install "jsonschema==4.23.0" - pip install "websockets==15.0.1" + uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - save_cache: paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Run prisma ./docker/entrypoint.sh command: | @@ -398,51 +350,52 @@ jobs: - run: name: Run tests command: | - pwd - ls - python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse" + uv run --no-sync python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse" no_output_timeout: 15m # Store test results - store_test_results: path: test-results auth_ui_unit_tests: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test working_directory: ~/project + environment: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-xdist==3.6.1" + uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" - run: - name: Run prisma ./docker/entrypoint.sh + name: Seed DB schema via prisma db push command: | set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh + uv run --no-sync litellm --skip_server_startup --use_prisma_db_push set -e + - run: + name: Generate Prisma Client + command: uv run --no-sync python -m prisma generate # Run pytest and generate JUnit XML report - run: name: Run tests command: | - pwd - ls - python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2 + uv run --no-sync python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2 no_output_timeout: 15m # Store test results @@ -451,50 +404,36 @@ jobs: litellm_router_testing: # Runs all tests with the "router" keyword docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project resource_class: large parallelism: 4 steps: - checkout - setup_google_dns + - install_uv - restore_cache: keys: - - v1-router-testing-deps-{{ checksum "requirements.txt" }} + - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "respx==0.22.0" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-xdist==3.6.1" - pip install "pytest-timeout==2.2.0" - pip install semantic_router --no-deps - pip install aurelio_sdk --no-deps + uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - - /home/circleci/.pyenv - - /home/circleci/.local - key: v1-router-testing-deps-{{ checksum "requirements.txt" }} + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | - pwd - ls TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ - --command="xargs python -m pytest \ + --command="xargs uv run --no-sync python -m pytest \ -v \ -k 'router' \ -n 4 \ @@ -508,119 +447,84 @@ jobs: litellm_router_unit_testing: # Runs all tests with the "router" keyword docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project resource_class: large steps: - checkout - setup_google_dns + - install_uv - restore_cache: keys: - - v1-router-unit-deps-{{ checksum "requirements.txt" }} + - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "respx==0.22.0" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install semantic_router --no-deps - pip install aurelio_sdk --no-deps - pip install "pytest-xdist==3.6.1" + uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - - /home/circleci/.pyenv - - /home/circleci/.local - key: v1-router-unit-deps-{{ checksum "requirements.txt" }} + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | - pwd - ls - python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4 + uv run --no-sync python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4 no_output_timeout: 15m # Store test results - store_test_results: path: test-results litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword docker: - - image: cimg/python:3.13.1 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project resource_class: medium steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - pip install wheel setuptools - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "respx==0.22.0" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" + uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | - pwd - ls - python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results - store_test_results: path: test-results llm_translation_testing: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project - resource_class: large + resource_class: xlarge steps: - checkout - setup_google_dns + - install_uv - restore_cache: keys: - - v1-llm-translation-deps-{{ checksum "requirements.txt" }} + - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pytest-xdist==3.6.1" - pip install "pytest-timeout==2.2.0" + uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - - /home/circleci/.pyenv - - /home/circleci/.local - key: v1-llm-translation-deps-{{ checksum "requirements.txt" }} + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | - pwd - ls # Add --timeout to kill hanging tests after 120s (2 min) # Add --durations=20 to show 20 slowest tests for debugging # Subdirectories with dedicated jobs (maintain this list as new jobs are added) @@ -631,7 +535,7 @@ jobs: for dir in "${IGNORE_DIRS[@]}"; do IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir" done - python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 + uv run --no-sync python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 --max-worker-restart=5 no_output_timeout: 15m # Store test results @@ -639,30 +543,24 @@ jobs: path: test-results realtime_translation_testing: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets" + uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run realtime tests command: | - pwd - ls # Add --timeout to kill hanging tests after 120s (2 min) # Add --durations=20 to show 20 slowest tests for debugging - python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread + uv run --no-sync python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread no_output_timeout: 15m - run: name: Rename the coverage files @@ -678,82 +576,24 @@ jobs: paths: - realtime_translation_coverage.xml - realtime_translation_coverage - mcp_testing: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pydantic==2.12.5" - pip install "mcp==1.26.0" - pip install "pytest-xdist==3.6.1" - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - pwd - ls - python -m pytest -vv tests/mcp_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml mcp_coverage.xml - mv .coverage mcp_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - mcp_coverage.xml - - mcp_coverage agent_testing: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pydantic==2.12.5" - pip install "a2a-sdk" + uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | - pwd - ls - python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m - run: name: Rename the coverage files @@ -771,38 +611,22 @@ jobs: - agent_coverage guardrails_testing: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pydantic==2.12.5" - pip install "boto3==1.42.80" - pip install "semantic_router==0.1.10" --no-deps - pip install aurelio_sdk - pip install "pytest-xdist==3.6.1" - pip install "pytest-timeout==2.2.0" + uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | - pwd - ls - LITELLM_LOG=WARNING python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread + LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread no_output_timeout: 15m - run: name: Rename the coverage files @@ -821,33 +645,22 @@ jobs: google_generate_content_endpoint_testing: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pydantic==2.12.5" + uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | - pwd - ls - python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5 + uv run --no-sync python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5 no_output_timeout: 15m - run: name: Rename the coverage files @@ -866,41 +679,30 @@ jobs: llm_responses_api_testing: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project resource_class: large steps: - checkout - setup_google_dns + - install_uv - restore_cache: keys: - - v1-llm-responses-deps-{{ checksum "requirements.txt" }} + - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pytest-xdist==3.6.1" + uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - - /home/circleci/.pyenv - - /home/circleci/.local - key: v1-llm-responses-deps-{{ checksum "requirements.txt" }} + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | - pwd - ls - python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8 + uv run --no-sync python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8 no_output_timeout: 15m # Store test results @@ -908,28 +710,22 @@ jobs: path: test-results ocr_testing: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" + uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | - pwd - ls - python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + uv run --no-sync python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 no_output_timeout: 15m - run: name: Rename the coverage files @@ -947,28 +743,22 @@ jobs: - ocr_coverage search_testing: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" + uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | - pwd - ls - python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + uv run --no-sync python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 no_output_timeout: 15m - run: name: Rename the coverage files @@ -984,120 +774,48 @@ jobs: paths: - search_coverage.xml - search_coverage - # Split litellm_mapped_tests into parallel jobs - litellm_mapped_tests_proxy_part1: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run proxy tests part 1 (high-volume directories) - command: | - prisma generate - export PYTHONUNBUFFERED=1 - python -m pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/client tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part1.xml --durations=10 -n 4 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_mapped_tests_proxy_part2: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run proxy tests part 2 (all other tests) - command: | - prisma generate - export PYTHONUNBUFFERED=1 - python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A - no_output_timeout: 15m - - store_test_results: - path: test-results litellm_mapped_enterprise_tests: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project resource_class: large steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest-mock==3.12.0" - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "hypercorn==0.17.3" - pip install "pydantic==2.12.5" - pip install "mcp==1.26.0" - pip install "requests-mock>=1.12.1" - pip install "responses==0.25.7" - pip install "pytest-xdist==3.6.1" - pip install "semantic_router==0.1.10" --no-deps - pip install aurelio_sdk - pip install "fastapi-offline==1.7.3" + uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - run: name: Run enterprise tests command: | - pwd - ls - prisma generate - python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4 + uv run --no-sync python -m prisma generate + uv run --no-sync python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4 no_output_timeout: 15m # Store test results - store_test_results: path: test-results batches_testing: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "respx==0.22.0" - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.133.0" - pip install "pytest-xdist==3.6.1" + uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | - pwd - ls - python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 + uv run --no-sync python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 no_output_timeout: 15m - run: name: Rename the coverage files @@ -1115,37 +833,22 @@ jobs: - batches_coverage litellm_utils_testing: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install numpydoc - pip install "respx==0.22.0" - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.133.0" - pip install pytest-mock - pip install "pytest-xdist==3.6.1" + uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | - pwd - ls - python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 + uv run --no-sync python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 no_output_timeout: 15m - run: name: Rename the coverage files @@ -1164,28 +867,22 @@ jobs: pass_through_unit_testing: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" + uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | - pwd - ls - python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + uv run --no-sync python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 no_output_timeout: 15m - run: name: Rename the coverage files @@ -1203,75 +900,46 @@ jobs: - pass_through_unit_tests_coverage image_gen_testing: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project resource_class: large steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "pytest-xdist==3.6.1" + uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | - pwd - ls - python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results - store_test_results: path: test-results logging_testing: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install pytest-mock - pip install "respx==0.22.0" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.133.0" - pip install "mlflow==2.17.2" - pip install "anthropic==0.54.0" - pip install "blockbuster==1.5.24" - pip install "pytest-xdist==3.6.1" - pip install "pytest-timeout==2.2.0" + uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | - pwd - ls - LITELLM_LOG=WARNING python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread + LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread no_output_timeout: 15m - run: name: Rename the coverage files @@ -1289,32 +957,22 @@ jobs: - logging_coverage audio_testing: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" + uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | - pwd - ls - python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m - run: name: Rename the coverage files @@ -1330,44 +988,75 @@ jobs: paths: - audio_coverage.xml - audio_coverage - installing_litellm_on_python: + redis_caching_unit_tests: docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - pip install python-dotenv - pip install pytest - pip install tiktoken - pip install aiohttp - pip install openai - pip install click - pip install "boto3==1.42.80" - pip install jinja2 - pip install "tokenizers==0.22.2" - pip install "uvloop==0.21.0" - pip install "fastuuid==0.14.0" - pip install jsonschema + uv sync --frozen --all-groups --all-extras --python 3.12 + - save_cache: + paths: + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + uv run --no-sync python -m pytest -vv \ + tests/local_testing/test_dual_cache.py \ + tests/local_testing/test_redis_batch_optimizations.py \ + tests/local_testing/test_router_utils.py \ + --cov=litellm --cov-report=xml \ + -x -s -v --junitxml=test-results/junit.xml \ + --durations=5 -n 2 \ + --reruns 2 --reruns-delay 1 + no_output_timeout: 20m + - run: + name: Rename the coverage files + command: | + mv coverage.xml redis_caching_coverage.xml + mv .coverage redis_caching_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - redis_caching_coverage.xml + - redis_caching_coverage + installing_litellm_on_python: + docker: + - *python312_image + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - run: name: Run tests command: | - pwd - ls - python -m pytest -vv tests/local_testing/test_basic_python_version.py + uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver" installing_litellm_on_python_3_13: docker: - - image: cimg/python:3.13.1 + - image: cimg/python:3.13.1@sha256:87b243ae80d154db75ce5e58af16c72c5dd4b1e23e5c7264a816e85e0c440c13 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} @@ -1377,27 +1066,49 @@ jobs: steps: - checkout - setup_google_dns + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip uv - pip install wheel setuptools - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "tomli==2.2.1" - pip install "mcp==1.26.0" + uv sync --frozen --all-groups --all-extras --python 3.13 - run: name: Run tests command: | - pwd - ls - python -m pytest -v tests/local_testing/test_basic_python_version.py + uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver" + + installing_litellm_on_python_v2_migration_resolver: + docker: + - *python312_image + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - setup_litellm_enterprise_pip + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" + - run: + name: Run v2 migration resolver proxy smoke test + command: | + uv run --no-sync python -m pytest -vv \ + tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver + helm_chart_testing: machine: - image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker + image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker resource_class: medium working_directory: ~/project @@ -1406,27 +1117,18 @@ jobs: - attach_workspace: at: ~/project - setup_google_dns - # Install Helm - - run: - name: Install Helm - command: | - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + - install_helm + - install_kind - # Install kind + # Install kubectl (pinned version with hardcoded checksum) - run: - name: Install Kind + name: Install kubectl v1.31.4 command: | - curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind - - # Install kubectl - - run: - name: Install kubectl - command: | - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - chmod +x kubectl - sudo mv kubectl /usr/local/bin/ + curl -sSLf -o /tmp/kubectl \ + https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl + echo "298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f /tmp/kubectl" | sha256sum -c - + chmod +x /tmp/kubectl + sudo mv /tmp/kubectl /usr/local/bin/ # Create kind cluster - run: @@ -1472,7 +1174,6 @@ jobs: # Run the helm tests helm test litellm --logs - helm test litellm --logs # Cleanup - run: @@ -1481,102 +1182,21 @@ jobs: kind delete cluster --name litellm-test when: always # This ensures cleanup runs even if previous steps fail - check_code_and_doc_quality: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project/litellm - - steps: - - checkout - - setup_google_dns - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - pip install ruff - pip install pylint - pip install pyright - pip install beautifulsoup4 - pip install . - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - - run: python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) - - run: ruff check ./litellm - # - run: python ./tests/documentation_tests/test_general_setting_keys.py - - run: python ./tests/code_coverage_tests/check_licenses.py - - run: python ./tests/code_coverage_tests/check_provider_folders_documented.py - - run: python ./tests/code_coverage_tests/router_code_coverage.py - - run: python ./tests/code_coverage_tests/test_chat_completion_imports.py - - run: python ./tests/code_coverage_tests/info_log_check.py - - run: python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py - - run: python ./tests/code_coverage_tests/test_ban_set_verbose.py - - run: python ./tests/code_coverage_tests/code_qa_check_tests.py - - run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py - - run: python ./tests/code_coverage_tests/test_proxy_types_import.py - - run: python ./tests/code_coverage_tests/callback_manager_test.py - - run: python ./tests/code_coverage_tests/recursive_detector.py - - run: python ./tests/code_coverage_tests/test_router_strategy_async.py - - run: python ./tests/code_coverage_tests/litellm_logging_code_coverage.py - - run: python ./tests/documentation_tests/test_env_keys.py - - run: python ./tests/documentation_tests/test_router_settings.py - - run: python ./tests/documentation_tests/test_api_docs.py - - run: python ./tests/code_coverage_tests/ensure_async_clients_test.py - - run: python ./tests/code_coverage_tests/enforce_llms_folder_style.py - - run: python ./tests/documentation_tests/test_circular_imports.py - - run: python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py - - run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py - - run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py - - run: python ./tests/code_coverage_tests/check_fastuuid_usage.py - - run: python ./tests/code_coverage_tests/memory_test.py - - run: helm lint ./deploy/charts/litellm-helm - db_migration_disable_update_check: machine: - image: ubuntu-2204:2023.10.1 + image: ubuntu-2204:2024.04.1 resource_class: medium working_directory: ~/project steps: - checkout - setup_google_dns - - run: - name: Install Python 3.9 - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.9 -y - conda activate myenv - python --version + - install_uv - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - pip install apscheduler - - run: - name: Install dockerize - command: | - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=litellm_test \ - -p 5432:5432 \ - postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres: + db_name: litellm_test - attach_workspace: at: ~/project - run: @@ -1598,9 +1218,9 @@ jobs: --config /app/config.yaml \ --port 4000 \ --use_prisma_db_push - - run: - name: Wait for schema seed to complete - command: dockerize -wait http://localhost:4001 -timeout 5m + - wait_for_service: + url: http://localhost:4001 + timeout: "300" - run: name: Stop schema seed container command: docker stop schema-seed && docker rm schema-seed @@ -1620,9 +1240,9 @@ jobs: litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - - run: - name: Wait for container to be ready - command: dockerize -wait http://localhost:4000 -timeout 1m + - wait_for_service: + url: http://localhost:4000 + timeout: "60" - run: name: Check container logs for expected message command: | @@ -1640,12 +1260,12 @@ jobs: - run: name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) command: | - python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 no_output_timeout: 15m build_and_test: machine: - image: ubuntu-2204:2023.10.1 + image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: @@ -1653,69 +1273,12 @@ jobs: - attach_workspace: at: ~/project - setup_google_dns - - run: - name: Install Python 3.9 - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.9 -y - conda activate myenv - python --version + - install_uv - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.133.0" - pip install pyarrow - pip install "boto3==1.42.80" - pip install langchain - pip install "langfuse>=2.0.0" - pip install "logfire==0.29.0" - pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema - pip install "httpx==0.28.1" - pip install "gunicorn==23.0.0" - pip install "anyio==4.8.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "PyGithub==1.59.1" - pip install "openai==1.100.1" - pip install "litellm[proxy]" - pip install "pytest-xdist==3.6.1" - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=circle_test \ - -p 5432:5432 \ - postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres - run: name: Load Docker Database Image command: | @@ -1758,24 +1321,17 @@ jobs: --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - - run: - name: Install curl - command: | - sudo apt-get update - sudo apt-get install -y curl - run: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run tests command: | - pwd - ls - python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests + uv run --no-sync python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests no_output_timeout: 15m # Store test results @@ -1783,82 +1339,18 @@ jobs: path: test-results e2e_openai_endpoints: machine: - image: ubuntu-2204:2023.10.1 + image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - - run: - name: Install Docker CLI (In case it's not already installed) - command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER - docker version - - run: - name: Install Python 3.10 - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.10 -y - conda activate myenv - python --version + - install_uv - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install "mypy==1.18.2" - pip install "jsonlines==4.0.0" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.133.0" - pip install pyarrow - pip install "boto3==1.42.80" - pip install langchain - pip install "langchain_mcp_adapters==0.0.5" - pip install "langfuse>=2.0.0" - pip install "logfire==0.29.0" - pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema - pip install "httpx==0.28.1" - pip install "gunicorn==23.0.0" - pip install "anyio==4.8.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "PyGithub==1.59.1" - pip install "openai==1.100.1" - # Run pytest and generate JUnit XML report - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=circle_test \ - -p 5432:5432 \ - postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres - attach_workspace: at: ~/project - run: @@ -1904,24 +1396,17 @@ jobs: --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - - run: - name: Install curl - command: | - sudo apt-get update - sudo apt-get install -y curl - run: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run tests command: | - pwd - ls - python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results @@ -1929,79 +1414,18 @@ jobs: path: test-results proxy_logging_guardrails_model_info_tests: machine: - image: ubuntu-2204:2023.10.1 + image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - - run: - name: Install Docker CLI (In case it's not already installed) - command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER - docker version - - run: - name: Install Python 3.9 - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.9 -y - conda activate myenv - python --version + - install_uv - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.133.0" - pip install pyarrow - pip install "boto3==1.42.80" - pip install langchain - pip install "langfuse>=2.0.0" - pip install "logfire==0.29.0" - pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema - pip install "httpx==0.28.1" - pip install "gunicorn==23.0.0" - pip install "anyio==4.8.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "PyGithub==1.59.1" - pip install "openai==1.100.1" - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=circle_test \ - -p 5432:5432 \ - postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres - attach_workspace: at: ~/project - run: @@ -2041,27 +1465,17 @@ jobs: --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run tests command: | - pwd - ls - python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Clean up first container - run: @@ -2094,17 +1508,17 @@ jobs: - run: name: Start outputting logs for second container - command: docker logs -f my-app-2 + command: docker logs -f my-app-3 background: true - - run: - name: Wait for second app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run second round of tests command: | - python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 no_output_timeout: 15m # Store test results @@ -2112,56 +1526,19 @@ jobs: path: test-results proxy_spend_accuracy_tests: machine: - image: ubuntu-2204:2023.10.1 + image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - - run: - name: Install Docker CLI (In case it's not already installed) - command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER - docker version - - run: - name: Install Python 3.9 - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.9 -y - conda activate myenv - python --version + - install_uv - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=circle_test \ - -p 5432:5432 \ - postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres + - start_redis - attach_workspace: at: ~/project - run: @@ -2171,15 +1548,18 @@ jobs: docker images | grep litellm-docker-database - run: name: Run Docker container - # intentionally give bad redis credentials here - # the OTEL test - should get this as a trace + # Point the proxy at the job-local Redis (start_redis) instead of the + # shared remote Redis. The Redis transaction buffer uses a single + # global pod-lock key (cronjob_lock:db_spend_update_job) and a single + # global buffer list (litellm_spend_update_buffer); sharing those + # across concurrent CI pipelines causes spend flushes to stall or + # land in the wrong DB, which is what makes this test flaky. command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e REDIS_HOST=$REDIS_HOST \ - -e REDIS_PASSWORD=$REDIS_PASSWORD \ - -e REDIS_PORT=$REDIS_PORT \ + -e REDIS_HOST=host.docker.internal \ + -e REDIS_PORT=6379 \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ @@ -2189,6 +1569,7 @@ jobs: -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ + -e PROXY_BATCH_WRITE_AT=2 \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \ @@ -2196,88 +1577,41 @@ jobs: --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - - run: - name: Install curl - command: | - sudo apt-get update - sudo apt-get install -y curl - run: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run tests command: | - pwd - ls - python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m - # Clean up first container - run: name: Stop and remove first container + when: always command: | docker stop my-app docker rm my-app + docker stop redis-cache + docker rm redis-cache proxy_multi_instance_tests: machine: - image: ubuntu-2204:2023.10.1 + image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - - run: - name: Install Docker CLI (In case it's not already installed) - command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER - docker version - - run: - name: Install Python 3.9 - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.9 -y - conda activate myenv - python --version + - install_uv - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=circle_test \ - -p 5432:5432 \ - postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres - attach_workspace: at: ~/project - run: @@ -2329,30 +1663,20 @@ jobs: --config /app/config.yaml \ --port 4001 \ --detailed_debug - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for instance 1 to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m - - run: - name: Wait for instance 2 to be ready - command: dockerize -wait http://localhost:4001 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" + - wait_for_service: + url: http://localhost:4001 + timeout: "300" - run: name: Run tests command: | - pwd - ls - python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Clean up first container # Store test results @@ -2361,62 +1685,18 @@ jobs: proxy_store_model_in_db_tests: machine: - image: ubuntu-2204:2023.10.1 + image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - - run: - name: Install Docker CLI (In case it's not already installed) - command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER - docker version - sudo systemctl restart docker - - run: - name: Install Python 3.9 - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.9 -y - conda activate myenv - python --version + - install_uv - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install "assemblyai==0.37.0" - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=circle_test \ - -p 5432:5432 \ - postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres - attach_workspace: at: ~/project - run: @@ -2442,27 +1722,17 @@ jobs: --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run tests command: | - pwd - ls - python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m - run: name: Stop and remove containers @@ -2478,48 +1748,23 @@ jobs: proxy_build_from_pip_tests: # Change from docker to machine executor machine: - image: ubuntu-2204:2023.10.1 + image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns # Remove Docker CLI installation since it's already available in machine executor - - run: - name: Install Python 3.13 - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.13 -y - conda activate myenv - python --version + - install_uv - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - pip install "pytest==7.3.1" "pytest-asyncio==0.21.1" "pytest-retry==1.6.3" \ - "pytest-mock==3.12.0" "mypy==1.18.2" aiohttp apscheduler + uv sync --frozen --all-groups --all-extras --python 3.12 - run: name: Build Docker image command: | docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip . - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=circle_test \ - -p 5432:5432 \ - postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: | - timeout 60s bash -c 'until docker exec postgres-db pg_isready -U postgres -d circle_test; do sleep 2; done' + - start_postgres - run: name: Run Docker container # intentionally give bad redis credentials here @@ -2550,25 +1795,17 @@ jobs: --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run tests command: | - python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + uv run --no-sync python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 no_output_timeout: 15m # Clean up first container - run: @@ -2581,78 +1818,18 @@ jobs: when: always proxy_pass_through_endpoint_tests: machine: - image: ubuntu-2204:2023.10.1 + image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - - run: - name: Install Python 3.10 - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.10 -y - conda activate myenv - python --version + - install_uv - run: name: Install Dependencies command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "google-cloud-aiplatform==1.133.0" - pip install aiohttp - pip install "openai==1.100.1" - pip install "assemblyai==0.37.0" - python -m pip install --upgrade pip - pip install "pydantic==2.12.5" - pip install "pytest==7.3.1" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install "boto3==1.42.80" - pip install "mypy==1.18.2" - pip install pyarrow - pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema - pip install "httpx==0.27.0" - pip install "anyio==4.8.0" - pip install "asyncio==3.4.3" - pip install "PyGithub==1.59.1" - pip install "google-cloud-aiplatform==1.59.0" - pip install "anthropic==0.54.0" - pip install "langchain_mcp_adapters==0.0.5" - pip install "langchain_openai==0.2.1" - pip install "langgraph==0.3.18" - pip install "fastuuid==0.13.5" - pip install -r requirements.txt - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=circle_test \ - -p 5432:5432 \ - postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres - attach_workspace: at: ~/project - run: @@ -2690,26 +1867,34 @@ jobs: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" # Add Ruby installation and testing before the existing Node.js and Python tests - run: name: Install Ruby and Bundler command: | - # Import GPG keys first - gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB || { - curl -sSL https://rvm.io/mpapis.asc | gpg --import - - curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - - } + # Clone RVM at pinned tag and verify the commit SHA matches the + # published tag before running its install script. + RVM_VERSION="1.29.12" + RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81" + git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm + RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)" + if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then + echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2 + exit 1 + fi - # Install Ruby version manager (RVM) - curl -sSL https://get.rvm.io | bash -s stable + # Import RVM signing keys (used by `rvm install` to verify Ruby tarballs) + gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB - # Source RVM from the correct location - source $HOME/.rvm/scripts/rvm + # Install RVM from the verified checkout. The install script + # sources `scripts/functions/installer` using paths relative to + # its own working directory, so it must be run from /tmp/rvm. + (cd /tmp/rvm && ./install --path "$HOME/.rvm") + source "$HOME/.rvm/scripts/rvm" - # Install Ruby 3.2.2 + # Install Ruby 3.2.2 (RVM verifies the tarball PGP signature) rvm install 3.2.2 rvm use 3.2.2 --default @@ -2724,38 +1909,38 @@ jobs: bundle install bundle exec rspec no_output_timeout: 30m - # New steps to run Node.js test + # Install Node.js directly from nodejs.org with SHA256 verification, + # instead of piping NodeSource's setup_18.x apt-repo installer into + # sudo bash (which runs a mutable upstream script unattended). - run: - name: Install Node.js + name: Install Node.js 18.20.8 command: | - export DEBIAN_FRONTEND=noninteractive - curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash - - sudo apt-get update - sudo apt-get install -y nodejs + NODE_VERSION="18.20.8" + NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz" + NODE_EXPECTED_SHA="5467ee62d6af1411d46b6a10e3fb5cacc92734dbcef465fea14e7b90993001c9" + curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" + echo "${NODE_EXPECTED_SHA} /tmp/${NODE_TARBALL}" | sha256sum -c - + sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /usr/local --strip-components=1 + rm -f "/tmp/${NODE_TARBALL}" node --version npm --version - run: - name: Install Node.js dependencies + name: Install Node.js test dependencies command: | - npm install @google-cloud/vertexai - npm install @google/generative-ai - npm install --save-dev jest + cd tests/pass_through_tests + npm ci - run: name: Run Vertex AI, Google AI Studio Node.js tests command: | - npx jest tests/pass_through_tests --verbose + cd tests/pass_through_tests + npx jest . --verbose no_output_timeout: 30m - run: name: Run tests command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - pwd - ls - python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results @@ -2764,60 +1949,18 @@ jobs: proxy_e2e_anthropic_messages_tests: machine: - image: ubuntu-2204:2023.10.1 + image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - - run: - name: Install Docker CLI (In case it's not already installed) - command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER - docker version - - run: - name: Install Python 3.10 - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.10 -y - conda activate myenv - python --version + - install_uv - run: name: Install Dependencies command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install "boto3==1.42.80" - pip install "httpx==0.27.0" - pip install "claude-agent-sdk" - pip install -r requirements.txt - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=circle_test \ - -p 5432:5432 \ - postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres - attach_workspace: at: ~/project - run: @@ -2848,137 +1991,24 @@ jobs: name: Start outputting logs command: docker logs -f my-app background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m + - wait_for_service: + url: http://localhost:4000 + timeout: "300" - run: name: Run Claude Agent SDK E2E Tests command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv export LITELLM_PROXY_URL="http://localhost:4000" export LITELLM_API_KEY="sk-1234" - pwd - ls - python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5 + uv run --no-sync python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 15m # Store test results - store_test_results: path: test-results - proxy_e2e_azure_batches_tests: - machine: - image: ubuntu-2204:2023.10.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - run: - name: Install Docker CLI - command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER - docker version - - run: - name: Install Python 3.12 - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.12 -y - conda activate myenv - python --version - - run: - name: Install Poetry - command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - pip install poetry - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=llmproxy \ - -e POSTGRES_PASSWORD=dbpassword9090 \ - -e POSTGRES_DB=litellm \ - -p 5432:5432 \ - postgres:15 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m - - run: - name: Install system dependencies - command: | - sudo apt-get update -y - sudo apt-get install -y libpq-dev - - run: - name: Install Dependencies - command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - poetry config virtualenvs.in-project true - poetry install --with dev,proxy-dev --extras "proxy" - poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity - - run: - name: Setup litellm-enterprise - command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - poetry run pip install --force-reinstall --no-deps -e enterprise/ - - run: - name: Generate Prisma client - command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - poetry run prisma generate --schema litellm/proxy/schema.prisma - - run: - name: Run Prisma migrations - command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm - cd litellm/proxy - poetry run prisma migrate deploy --schema schema.prisma - cd ../.. - - run: - name: Run Azure Batch E2E Tests - command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm - export USE_LOCAL_LITELLM=true - export USE_MOCK_MODELS=true - export USE_STATE_TRACKER=true - export LITELLM_LOG=DEBUG - poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \ - -vv -s -k "test_e2e_managed_batch" \ - --tb=short \ - --maxfail=3 \ - --durations=10 \ - --junitxml=test-results/junit.xml - no_output_timeout: 15m - upload-coverage: docker: - - image: cimg/python:3.9 + - *python312_image steps: - checkout - attach_workspace: @@ -2991,126 +2021,18 @@ jobs: ls -la echo "\nContents of tests/llm_translation:" ls -la tests/llm_translation + - install_uv - run: name: Combine Coverage command: | - python -m venv venv - . venv/bin/activate - pip install coverage - coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage - coverage xml + uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage + uv tool run --from 'coverage[toml]==7.10.6' coverage xml - codecov/upload: file: ./coverage.xml - publish_proxy_extras: - docker: - - image: cimg/python:3.8 - working_directory: ~/project/litellm-proxy-extras - environment: - TWINE_USERNAME: __token__ - - steps: - - checkout: - path: ~/project - - - run: - name: Check if litellm-proxy-extras dir or pyproject.toml was modified - command: | - echo "Install TOML package." - python -m pip install toml - # Get current version from pyproject.toml - CURRENT_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - - # Get last published version from PyPI - LAST_VERSION=$(curl -s https://pypi.org/pypi/litellm-proxy-extras/json | python -c "import json, sys; print(json.load(sys.stdin)['info']['version'])") - - echo "Current version: $CURRENT_VERSION" - echo "Last published version: $LAST_VERSION" - - # Compare versions using Python's packaging.version - VERSION_COMPARE=$(python -c "from packaging import version; print(1 if version.parse('$CURRENT_VERSION') < version.parse('$LAST_VERSION') else 0)") - - echo "Version compare: $VERSION_COMPARE" - if [ "$VERSION_COMPARE" = "1" ]; then - echo "Error: Current version ($CURRENT_VERSION) is less than last published version ($LAST_VERSION)" - exit 1 - fi - - # If versions are equal or current is greater, check contents - pip download --no-deps litellm-proxy-extras==$LAST_VERSION -d /tmp - - echo "Contents of /tmp directory:" - ls -la /tmp - - # Find the downloaded file (could be .whl or .tar.gz) - DOWNLOADED_FILE=$(ls /tmp/litellm_proxy_extras-*) - echo "Downloaded file: $DOWNLOADED_FILE" - - # Extract based on file extension - if [[ "$DOWNLOADED_FILE" == *.whl ]]; then - echo "Extracting wheel file..." - unzip -q "$DOWNLOADED_FILE" -d /tmp/extracted - EXTRACTED_DIR="/tmp/extracted" - else - echo "Extracting tar.gz file..." - tar -xzf "$DOWNLOADED_FILE" -C /tmp - EXTRACTED_DIR="/tmp/litellm_proxy_extras-$LAST_VERSION" - fi - - echo "Contents of extracted package:" - ls -R "$EXTRACTED_DIR" - - # Compare contents - if ! diff -r "$EXTRACTED_DIR/litellm_proxy_extras" ./litellm_proxy_extras; then - if [ "$CURRENT_VERSION" = "$LAST_VERSION" ]; then - echo "Error: Changes detected in litellm-proxy-extras but version was not bumped" - echo "Current version: $CURRENT_VERSION" - echo "Last published version: $LAST_VERSION" - echo "Changes:" - diff -r "$EXTRACTED_DIR/litellm_proxy_extras" ./litellm_proxy_extras - exit 1 - fi - else - echo "No changes detected in litellm-proxy-extras. Skipping PyPI publish." - circleci step halt - fi - - - run: - name: Get new version - command: | - NEW_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV - - - run: - name: Check if versions match - command: | - cd ~/project - # Check pyproject.toml - CURRENT_VERSION=$(python -c "import toml; dep = toml.load('pyproject.toml')['tool']['poetry']['dependencies']['litellm-proxy-extras']; print(dep['version'] if isinstance(dep, dict) else dep)") - if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then - echo "Error: Version in pyproject.toml ($CURRENT_VERSION) doesn't match new version ($NEW_VERSION)" - exit 1 - fi - - # Check requirements.txt - REQ_VERSION=$(grep -oP 'litellm-proxy-extras==\K[0-9.]+' requirements.txt) - if [ "$REQ_VERSION" != "$NEW_VERSION" ]; then - echo "Error: Version in requirements.txt ($REQ_VERSION) doesn't match new version ($NEW_VERSION)" - exit 1 - fi - - - run: - name: Publish to PyPI - command: | - echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc - python -m pip install --upgrade pip build twine setuptools wheel - rm -rf build dist - python -m build - twine upload --verbose dist/* - ui_build: docker: - - image: cimg/node:20.19 + - image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} @@ -3152,7 +2074,7 @@ jobs: ui_unit_tests: docker: - - image: cimg/node:20.19 + - image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} @@ -3182,6 +2104,124 @@ jobs: CI=true npm run test -- --run \ --pool forks --poolOptions.forks.maxForks=8 + e2e_ui_testing: + docker: + - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: e2euser + POSTGRES_PASSWORD: e2epassword + POSTGRES_DB: litellm_e2e + resource_class: large + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" + CI: "true" + steps: + - checkout + - setup_google_dns + - install_uv + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - run: + name: Install Python dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma + - save_cache: + key: v1-uv-cache-{{ checksum "uv.lock" }} + paths: + - ~/.cache/uv + - restore_cache: + keys: + - ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - run: + name: Install Node dependencies and Playwright + command: | + cd ui/litellm-dashboard + npm ci + npx playwright install chromium --with-deps + - save_cache: + key: ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/node_modules + - run: + name: Build UI from source + # Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`. + # GNU cp (used on CircleCI's Ubuntu image) interprets that as "copy the + # source directory as a child of the destination" when the destination + # already exists — silently creating `_experimental/out/out/` instead of + # replacing the served bundle. The proxy continued serving whatever was + # checked into `_experimental/out/*`, so this job was effectively testing + # the pre-build bundle on every run. Replace-and-move guarantees the + # freshly built bundle is what the proxy actually serves. + command: | + cd ui/litellm-dashboard + npm run build + rm -rf ../../litellm/proxy/_experimental/out + mv out ../../litellm/proxy/_experimental/out + # Restructure HTML so extensionless routes work (login.html -> login/index.html) + find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do + d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" + done + - wait_for_service: + url: tcp://localhost:5432 + timeout: "30" + - run: + name: Push Prisma schema + command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + - run: + name: Seed database + command: | + PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ + -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql + - run: + name: Start mock LLM server + command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py + background: true + - run: + name: Start LiteLLM proxy + environment: + LITELLM_MASTER_KEY: "sk-1234" + MOCK_LLM_URL: "http://127.0.0.1:8090/v1" + DISABLE_SCHEMA_UPDATE: "true" + SERVER_ROOT_PATH: "" + PROXY_LOGOUT_URL: "" + command: | + uv run --no-sync python -m litellm.proxy.proxy_cli \ + --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \ + --port 4000 + background: true + - run: + name: Wait for proxy to be ready + command: | + for i in $(seq 1 60); do + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true) + if [ "$HTTP_CODE" = "200" ]; then + echo "Proxy is ready" + exit 0 + fi + sleep 2 + done + echo "Proxy failed to start" + exit 1 + - run: + name: Run Playwright E2E tests + command: | + cd ui/litellm-dashboard + npx playwright test --config e2e_tests/playwright.config.ts + no_output_timeout: 10m + - store_artifacts: + path: ui/litellm-dashboard/test-results + destination: e2e-test-results + - store_artifacts: + path: ui/litellm-dashboard/playwright-report + destination: e2e-playwright-report + build_docker_database_image: machine: image: ubuntu-2204:2024.04.1 @@ -3207,144 +2247,10 @@ jobs: paths: - litellm-docker-database.tar.zst - e2e_ui_testing: - machine: - image: ubuntu-2204:2023.10.1 - resource_class: large - working_directory: ~/project - parameters: - browser: - type: string - steps: - - checkout - - setup_google_dns - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Install Dependencies - command: | - npm install -D @playwright/test - - run: - name: Install Playwright Browsers - command: | - npx playwright install - - run: - name: Run Docker container - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \ - -e LITELLM_MASTER_KEY="sk-1234" \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e UI_USERNAME="admin" \ - -e UI_PASSWORD="gm" \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - --name litellm-docker-database-<< parameters.browser >> \ - -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start outputting logs - command: docker logs -f litellm-docker-database-<< parameters.browser >> - background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m - - run: - name: Run Playwright Tests - command: | - npx playwright test \ - --project << parameters.browser >> \ - --config ui/litellm-dashboard/e2e_tests/playwright.config.ts \ - --reporter=html \ - --output=test-results - no_output_timeout: 15m - - store_artifacts: - path: test-results - destination: playwright-results - - - store_artifacts: - path: playwright-report - destination: playwright-report - - prisma_schema_sync: - machine: - image: ubuntu-2204:2023.10.1 - resource_class: medium - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - attach_workspace: - at: ~/project - - run: - name: Install dockerize - command: | - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=litellm_schema_sync \ - -p 5432:5432 \ - postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Run schema sync via prisma db push - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \ - -e LITELLM_MASTER_KEY="sk-1234" \ - --name schema-sync \ - --add-host=host.docker.internal:host-gateway \ - -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --use_prisma_db_push - - run: - name: Start outputting logs - command: docker logs -f schema-sync - background: true - - run: - name: Wait for proxy to be ready (schema sync complete) - command: dockerize -wait http://localhost:4000 -timeout 5m - - run: - name: Stop schema sync container - command: docker stop schema-sync - test_bad_database_url: machine: - image: ubuntu-2204:2023.10.1 + image: ubuntu-2204:2024.04.1 resource_class: medium working_directory: ~/project steps: @@ -3352,25 +2258,7 @@ jobs: - attach_workspace: at: ~/project - setup_google_dns - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=circle_test \ - -p 5432:5432 \ - postgres:14 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m + - start_postgres - run: name: Load Docker Database Image command: | @@ -3402,344 +2290,112 @@ jobs: fi workflows: - version: 2 build_and_test: jobs: - using_litellm_on_windows: - filters: - branches: - only: - - main - - /litellm_.*/ - - mypy_linting: - filters: - branches: - only: - - main - - /litellm_.*/ - - semgrep: - filters: + filters: &main_branches branches: only: - main - /litellm_.*/ - local_testing_part1: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - local_testing_part2: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - langfuse_logging_unit_tests: - filters: - branches: - only: - - main - - /litellm_.*/ - - main - - /litellm_.*/ - - main - - /litellm_.*/ - - main - - /litellm_.*/ - - main - - /litellm_.*/ - - main - - /litellm_.*/ + filters: *main_branches - litellm_assistants_api_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - litellm_router_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - litellm_router_unit_testing: - filters: - branches: - only: - - main - - /litellm_.*/ - - check_code_and_doc_quality: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - ui_build: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - ui_unit_tests: requires: - ui_build - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - auth_ui_unit_tests: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - build_docker_database_image: - filters: - branches: - only: - - main - - /litellm_.*/ - - prisma_schema_sync: - requires: - - build_docker_database_image - filters: - branches: - only: - - main - - /litellm_.*/ - # - e2e_ui_testing: - # name: e2e_ui_testing_chromium - # browser: chromium - # context: e2e_ui_tests - # requires: - # - ui_build - # - build_docker_database_image - # - prisma_schema_sync - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - e2e_ui_testing: - # name: e2e_ui_testing_firefox - # browser: firefox - # context: e2e_ui_tests - # requires: - # - ui_build - # - build_docker_database_image - # - prisma_schema_sync - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ + filters: *main_branches + - e2e_ui_testing: + filters: *main_branches - build_and_test: requires: - build_docker_database_image - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - e2e_openai_endpoints: requires: - build_docker_database_image - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - proxy_logging_guardrails_model_info_tests: requires: - build_docker_database_image - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - proxy_spend_accuracy_tests: requires: - build_docker_database_image - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - proxy_multi_instance_tests: requires: - build_docker_database_image - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - proxy_store_model_in_db_tests: requires: - build_docker_database_image - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - proxy_build_from_pip_tests: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - proxy_pass_through_endpoint_tests: requires: - build_docker_database_image - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - proxy_e2e_anthropic_messages_tests: requires: - build_docker_database_image - filters: - branches: - only: - - main - - /litellm_.*/ - - proxy_e2e_azure_batches_tests: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - llm_translation_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - realtime_translation_testing: - filters: - branches: - only: - - main - - /litellm_.*/ - - mcp_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - agent_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - guardrails_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - google_generate_content_endpoint_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - llm_responses_api_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - ocr_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - search_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - litellm_mapped_enterprise_tests: - filters: - branches: - only: - - main - - /litellm_.*/ - - litellm_mapped_tests_proxy_part1: - filters: - branches: - only: - - main - - /litellm_.*/ - - litellm_mapped_tests_proxy_part2: - filters: - branches: - only: - - main - - /litellm_.*/ - - main - - /litellm_.*/ - - main - - /litellm_.*/ - - main - - /litellm_.*/ - - main - - /litellm_.*/ - - main - - /litellm_.*/ + filters: *main_branches - batches_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - litellm_utils_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - pass_through_unit_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - image_gen_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - logging_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - audio_testing: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches + - redis_caching_unit_tests: + filters: *main_branches - upload-coverage: requires: - realtime_translation_testing - - mcp_testing - agent_testing - google_generate_content_endpoint_testing - guardrails_testing - ocr_testing - search_testing - - litellm_mapped_tests_proxy_part1 - - litellm_mapped_tests_proxy_part2 - litellm_mapped_enterprise_tests - batches_testing - litellm_utils_testing @@ -3747,6 +2403,7 @@ workflows: - image_gen_testing - logging_testing - audio_testing + - redis_caching_unit_tests - langfuse_logging_unit_tests - local_testing_part1 - local_testing_part2 @@ -3754,42 +2411,18 @@ workflows: - db_migration_disable_update_check: requires: - build_docker_database_image - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - installing_litellm_on_python: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - installing_litellm_on_python_3_13: - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches + - installing_litellm_on_python_v2_migration_resolver: + filters: *main_branches - helm_chart_testing: requires: - build_docker_database_image - filters: - branches: - only: - - main - - /litellm_.*/ + filters: *main_branches - test_bad_database_url: requires: - build_docker_database_image - filters: - branches: - only: - - main - - /litellm_.*/ - - publish_proxy_extras: - filters: - branches: - only: - - main - - /litellm_release_day_.*/ + filters: *main_branches diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt deleted file mode 100644 index be12ab2d0f..0000000000 --- a/.circleci/requirements.txt +++ /dev/null @@ -1,21 +0,0 @@ -# used by CI/CD testing -openai==1.100.1 -python-dotenv -tiktoken -importlib_metadata -cohere -redis==5.2.1 -redisvl==0.4.1 -anthropic -orjson==3.10.15 # fast /embedding responses -pydantic==2.12.5 -google-cloud-aiplatform==1.133.0 -google-cloud-iam==2.19.1 -fastapi-sso==0.16.0 -uvloop==0.21.0 -mcp==1.26.0 # for MCP server -semantic_router==0.1.10 # for auto-routing with litellm -fastuuid==0.14.0 -responses==0.25.7 # for proxy client tests -pytest-retry==1.6.3 # for automatic test retries -litellm-proxy-extras # for prisma migrations \ No newline at end of file diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index f1ae30e67d..9377cbeb0c 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -27,6 +27,10 @@ on: required: false type: number default: 10 + artifact-name: + description: "Unique name for the coverage artifact (must be unique per run)" + required: true + type: string permissions: contents: read @@ -47,37 +51,30 @@ jobs: with: python-version: "3.12" - - name: Install Poetry - run: pip install 'poetry==2.3.2' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - - name: Cache Poetry dependencies + - name: Cache uv dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | - ~/.cache/pypoetry - ~/.cache/pip + ~/.cache/uv .venv - key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-poetry- + ${{ runner.os }}-uv- - name: Install dependencies run: | - poetry config virtualenvs.in-project true - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install google-genai==1.22.0 \ - google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - - - name: Setup litellm-enterprise - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router - name: Generate Prisma client env: PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | - poetry run pip install nodejs-wheel-binaries==24.13.1 - poetry run prisma generate --schema litellm/proxy/schema.prisma + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Run tests env: @@ -86,11 +83,53 @@ jobs: WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} run: | - poetry run pytest ${TEST_PATH:?} \ + uv run --no-sync pytest ${TEST_PATH:?} \ --tb=short -vv \ --maxfail="${MAX_FAILURES}" \ -n "${WORKERS}" \ --reruns "${RERUNS}" \ --reruns-delay 1 \ --dist=loadscope \ - --durations=20 + --durations=20 \ + --cov=litellm \ + --cov-report=xml:coverage.xml \ + --cov-config=pyproject.toml + + - name: Save coverage report + if: always() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: coverage.xml + retention-days: 1 + + upload-coverage: + name: Upload coverage to Codecov + needs: run + if: always() + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + pull-requests: write + + steps: + - name: Checkout code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Download coverage report + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + with: + pattern: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: coverage-reports + merge-multiple: true + + - name: Upload to Codecov + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 + with: + use_oidc: true + directory: coverage-reports + root_dir: ${{ github.workspace }} + fail_ci_if_error: false diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml index d53a9e8822..8c47b6d766 100644 --- a/.github/workflows/_test-unit-services-base.yml +++ b/.github/workflows/_test-unit-services-base.yml @@ -27,55 +27,44 @@ on: required: false type: number default: 10 - enable-redis: - description: "Pass Redis Cloud credentials to tests via REDIS_HOST/PORT/PASSWORD env vars" - required: false - type: boolean - default: false enable-postgres: description: "Start a local Postgres service container and run Prisma migrations" required: false type: boolean default: false - secrets: - REDIS_HOST: + dist: + description: "pytest-xdist distribution mode (loadscope|load|worksteal|loadfile|no)" required: false - REDIS_PORT: - required: false - REDIS_PASSWORD: - required: false - DATABASE_URL: - required: false - POSTGRES_USER: - required: false - POSTGRES_PASSWORD: + type: string + default: "loadscope" + artifact-name: + description: "Unique name for the coverage artifact (must be unique per run)" required: false + type: string + default: "run" permissions: contents: read +# The postgres service container below is spawned per-job on localhost and +# destroyed with the job. Nothing outside the runner can reach it. The +# user/password/database here are not secrets — they're bootstrap values +# for a throwaway container — so we hardcode them instead of attaching +# every matrix shard to a GHA environment just to read three "secrets" +# (which also produces a "temporarily deployed to …" notification on the +# PR timeline per shard per push). jobs: run: name: Run tests runs-on: ubuntu-latest timeout-minutes: ${{ inputs.timeout-minutes }} - # Environment is derived from the enable-* flags, not caller-controllable. - # This prevents callers from passing arbitrary environment names to bypass secret scoping. - # Note: Postgres service container always starts (GHA limitation), so any Redis job - # also needs Postgres secrets → uses integration-redis-postgres, not integration-redis. - environment: >- - ${{ - inputs.enable-redis && 'integration-redis-postgres' || - inputs.enable-postgres && 'integration-postgres' || - '' - }} services: postgres: image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14 env: - POSTGRES_USER: ${{ secrets.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + POSTGRES_USER: litellm + POSTGRES_PASSWORD: litellm POSTGRES_DB: litellm_test ports: - 5432:5432 @@ -95,44 +84,37 @@ jobs: with: python-version: "3.12" - - name: Install Poetry - run: pip install 'poetry==2.3.2' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - - name: Cache Poetry dependencies + - name: Cache uv dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | - ~/.cache/pypoetry - ~/.cache/pip + ~/.cache/uv .venv - key: ${{ runner.os }}-poetry-services-${{ hashFiles('poetry.lock') }} + key: ${{ runner.os }}-uv-services-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-poetry-services- + ${{ runner.os }}-uv-services- - name: Install dependencies run: | - poetry config virtualenvs.in-project true - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install google-genai==1.22.0 \ - google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - - - name: Setup litellm-enterprise - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router - name: Generate Prisma client env: PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | - poetry run pip install nodejs-wheel-binaries==24.13.1 - poetry run prisma generate --schema litellm/proxy/schema.prisma + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Run Prisma migrations if: ${{ inputs.enable-postgres }} env: - DATABASE_URL: ${{ secrets.DATABASE_URL }} + DATABASE_URL: "postgresql://litellm:litellm@localhost:5432/litellm_test" run: | - poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - name: Run tests env: @@ -140,25 +122,68 @@ jobs: MAX_FAILURES: ${{ inputs.max-failures }} WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} - DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }} - REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }} - REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }} - REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }} + DIST: ${{ inputs.dist }} + DATABASE_URL: ${{ inputs.enable-postgres && 'postgresql://litellm:litellm@localhost:5432/litellm_test' || '' }} run: | if [ "${WORKERS}" = "0" ]; then - poetry run pytest ${TEST_PATH:?} \ + uv run --no-sync pytest ${TEST_PATH:?} \ --tb=short -vv \ --maxfail="${MAX_FAILURES}" \ --reruns "${RERUNS}" \ --reruns-delay 1 \ - --durations=20 + --durations=20 \ + --cov=litellm \ + --cov-report=xml:coverage.xml \ + --cov-config=pyproject.toml else - poetry run pytest ${TEST_PATH:?} \ + uv run --no-sync pytest ${TEST_PATH:?} \ --tb=short -vv \ --maxfail="${MAX_FAILURES}" \ -n "${WORKERS}" \ --reruns "${RERUNS}" \ --reruns-delay 1 \ - --dist=loadscope \ - --durations=20 + --dist="${DIST}" \ + --durations=20 \ + --cov=litellm \ + --cov-report=xml:coverage.xml \ + --cov-config=pyproject.toml fi + + - name: Save coverage report + if: always() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: coverage.xml + retention-days: 1 + + upload-coverage: + name: Upload coverage to Codecov + needs: run + if: always() + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + pull-requests: write + + steps: + - name: Checkout code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Download coverage report + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + with: + pattern: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: coverage-reports + merge-multiple: true + + - name: Upload to Codecov + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 + with: + use_oidc: true + directory: coverage-reports + root_dir: ${{ github.workspace }} + fail_ci_if_error: false diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml index 60e8993621..1c6c318c71 100644 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -17,12 +17,13 @@ jobs: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: persist-credentials: false - - name: Install Dependencies - run: | - pip install 'aiohttp==3.13.3' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - name: Update JSON Data run: | - python ".github/workflows/auto_update_price_and_context_window_file.py" + uv run --frozen --with 'aiohttp==3.13.3' python ".github/workflows/auto_update_price_and_context_window_file.py" - name: Create Pull Request run: | git add model_prices_and_context_window.json diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 2ae01823a9..68ab397d82 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -48,7 +48,21 @@ jobs: const cosignSection = [ `## Verify Docker Image Signature`, ``, - `All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying:`, + `All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit \`0112e53\`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).`, + ``, + `**Verify using the pinned commit hash (recommended):**`, + ``, + `A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:`, + ``, + '```bash', + `cosign verify \\`, + ` --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \\`, + ` ghcr.io/berriai/litellm:${tag}`, + '```', + ``, + `**Verify using the release tag (convenience):**`, + ``, + `Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:`, ``, '```bash', `cosign verify \\`, @@ -88,6 +102,17 @@ jobs: body: updatedBody, draft: false, }); + } catch (error) { core.setFailed(error.message); } + + create-branch: + name: Create Release Branch + needs: release + permissions: + contents: write + uses: ./.github/workflows/create-release-branch.yml + with: + tag: ${{ inputs.tag }} + commit_hash: ${{ inputs.commit_hash }} diff --git a/.github/workflows/llm-translation-testing.yml b/.github/workflows/llm-translation-testing.yml index 922013c4b5..8d9d52f4e5 100644 --- a/.github/workflows/llm-translation-testing.yml +++ b/.github/workflows/llm-translation-testing.yml @@ -29,28 +29,27 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: "3.11" + python-version: "3.12" - - name: Install Poetry - run: | - pip install 'poetry==2.3.2' - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + enable-cache: false - - name: Restore Poetry dependencies cache - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 + - name: Restore uv dependencies cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | - ~/.cache/pypoetry + ~/.cache/uv .venv - key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-poetry- + ${{ runner.os }}-uv- - name: Install dependencies run: | - poetry install --with dev - poetry run pip install 'pytest-xdist==3.8.0' 'pytest-timeout==2.4.0' + uv sync --frozen - name: Create test results directory run: mkdir -p test-results diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml index 8f675bb307..d60254a0ac 100644 --- a/.github/workflows/publish_to_pypi.yml +++ b/.github/workflows/publish_to_pypi.yml @@ -24,10 +24,22 @@ jobs: with: python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + enable-cache: false + - name: Check litellm version on PyPI id: check-litellm run: | - VERSION=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/') + VERSION=$(python - <<'PY' + import tomllib + + with open("pyproject.toml", "rb") as f: + print(tomllib.load(f)["project"]["version"]) + PY + ) echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "Checking if litellm $VERSION exists on PyPI..." @@ -42,43 +54,46 @@ jobs: - name: Sanity check proxy-extras version run: | - # Read pinned version from requirements.txt - REQ_VERSION=$(grep -oP 'litellm-proxy-extras==\K[0-9.]+' requirements.txt) - if [ -z "$REQ_VERSION" ]; then - echo "::error::Could not find litellm-proxy-extras version in requirements.txt" - exit 1 - fi - echo "requirements.txt pins litellm-proxy-extras==$REQ_VERSION" + # Read pinned version from project optional dependencies + PYPROJECT_VERSION=$(python3 - <<'PY' + import sys + import tomllib - # Read pinned version from pyproject.toml dependency - PYPROJECT_VERSION=$(python3 -c " - import re - with open('pyproject.toml') as f: - content = f.read() - match = re.search(r'litellm-proxy-extras\s*=\s*\{version\s*=\s*\"([^\"]+)\"', content) - if match: - print(match.group(1).lstrip('^~>=')) - else: - import sys - print('::error::Could not find litellm-proxy-extras dependency in pyproject.toml', file=sys.stderr) + with open("pyproject.toml", "rb") as f: + proxy_requirements = tomllib.load(f)["project"]["optional-dependencies"]["proxy"] + + version = None + for requirement in proxy_requirements: + normalized = requirement.split(";", 1)[0].strip() + if not normalized.startswith("litellm-proxy-extras"): + continue + parts = normalized.split("==", 1) + if len(parts) == 2 and parts[0].strip() == "litellm-proxy-extras": + candidate = parts[1].strip() + if candidate: + version = candidate + break + + if version is None: + print( + "::error::Could not find an exact litellm-proxy-extras pin in project.optional-dependencies.proxy", + file=sys.stderr, + ) sys.exit(1) - ") + + print(version) + PY + ) echo "pyproject.toml pins litellm-proxy-extras version: $PYPROJECT_VERSION" - # Check that both pinned versions match - if [ "$REQ_VERSION" != "$PYPROJECT_VERSION" ]; then - echo "::error::Version mismatch: requirements.txt has $REQ_VERSION but pyproject.toml has $PYPROJECT_VERSION" - exit 1 - fi - # Check that the pinned version exists on PyPI - echo "Checking if litellm-proxy-extras $REQ_VERSION exists on PyPI..." - HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$REQ_VERSION/json") + echo "Checking if litellm-proxy-extras $PYPROJECT_VERSION exists on PyPI..." + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$PYPROJECT_VERSION/json") if [ "$HTTP_STATUS" != "200" ]; then - echo "::error::litellm-proxy-extras $REQ_VERSION is not published on PyPI yet. Publish it before releasing litellm." + echo "::error::litellm-proxy-extras $PYPROJECT_VERSION is not published on PyPI yet. Publish it before releasing litellm." exit 1 fi - echo "litellm-proxy-extras $REQ_VERSION exists on PyPI. Sanity check passed." + echo "litellm-proxy-extras $PYPROJECT_VERSION exists on PyPI. Sanity check passed." publish-litellm: name: Publish litellm to PyPI @@ -100,16 +115,19 @@ jobs: with: python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + enable-cache: false + - name: Copy model prices backup run: cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json - - name: Install build tools - run: python -m pip install --upgrade pip build==1.4.2 - - name: Build package run: | rm -rf build dist - python -m build + uv build - name: Verify build artifacts env: @@ -129,8 +147,7 @@ jobs: - name: Validate package metadata run: | - pip install twine==6.2.0 - twine check dist/* + uv tool run --from 'twine==6.2.0' twine check dist/* - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 diff --git a/.github/workflows/scan_duplicate_issues.yml b/.github/workflows/scan_duplicate_issues.yml index 222ff11f30..ab0ac2aa3a 100644 --- a/.github/workflows/scan_duplicate_issues.yml +++ b/.github/workflows/scan_duplicate_issues.yml @@ -29,7 +29,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: "3.13" + python-version: "3.12" - name: Scan for duplicate issues env: diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 5bb85716a1..b5e45a38cf 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -2,7 +2,11 @@ name: LiteLLM Linting on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" permissions: contents: read @@ -24,26 +28,28 @@ jobs: with: python-version: "3.12" - - name: Install Poetry - run: pip install 'poetry==2.3.2' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - name: Clean Python cache run: | find . -type d -name "__pycache__" -exec rm -rf {} + || true find . -name "*.pyc" -delete || true - - name: Check poetry.lock is up to date + - name: Check uv.lock is up to date run: | - poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1) + uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1) - name: Install dependencies run: | - poetry install --with dev + uv sync --frozen - name: Check Black formatting run: | cd litellm - poetry run black --check --exclude '/enterprise/' . + uv run --no-sync black --check --exclude '/enterprise/' . cd .. - name: Debug - Check file state @@ -58,28 +64,28 @@ jobs: - name: Run Ruff linting run: | cd litellm - poetry run ruff check . + uv run --no-sync ruff check . cd .. - name: Print OpenAI version run: | - poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')" + uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')" - name: Run MyPy type checking run: | cd litellm - poetry run mypy . + uv run --no-sync mypy . cd .. - name: Check for circular imports run: | cd litellm - poetry run python ../tests/documentation_tests/test_circular_imports.py + uv run --no-sync python ../tests/documentation_tests/test_circular_imports.py cd .. - name: Check import safety run: | - poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) + uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) secret-scan: runs-on: ubuntu-latest @@ -98,18 +104,21 @@ jobs: with: python-version: "3.12" + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + - name: Run secret scan test run: | - pip install 'pytest==9.0.2' - pytest tests/litellm/test_no_hardcoded_secrets.py -v + uv run --frozen --with 'pytest==9.0.2' pytest tests/litellm/test_no_hardcoded_secrets.py -v - name: Run ggshield secret scan env: GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} run: | if [ -n "$GITGUARDIAN_API_KEY" ]; then - pip install 'ggshield==1.48.0' - ggshield secret scan repo . + uv tool run --from 'ggshield==1.48.0' ggshield secret scan repo . else echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan" fi diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml deleted file mode 100644 index dafabe4d83..0000000000 --- a/.github/workflows/test-litellm-matrix.yml +++ /dev/null @@ -1,214 +0,0 @@ -name: LiteLLM Unit Tests (Matrix) - -on: - pull_request: - branches: [main] - -permissions: - contents: read - -# Cancel in-progress runs for the same PR -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 20 # Increased from 15 to 20 - strategy: - fail-fast: false - matrix: - test-group: - # tests/test_litellm split by subdirectory (~560 files total) - # Vertex AI tests separated for better isolation (prevent auth/env pollution) - - name: "llms-vertex" - path: "tests/test_litellm/llms/vertex_ai" - workers: 1 - reruns: 2 - - name: "llms-other" - path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" - workers: 2 - reruns: 2 - # tests/test_litellm/proxy split by subdirectory (~180 files total) - - name: "proxy-guardrails" - path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers" - workers: 2 - reruns: 2 - - name: "proxy-core" - path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine" - workers: 2 - reruns: 2 - - name: "proxy-misc" - path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py" - workers: 2 - reruns: 2 - - name: "integrations" - path: "tests/test_litellm/integrations" - workers: 2 - reruns: 3 # Integration tests tend to be flakier - - name: "core-utils" - path: "tests/test_litellm/litellm_core_utils" - workers: 2 - reruns: 1 - - name: "other-1" - # responses (5942) + caching (1723) + types (819) ≈ 8.5k lines - path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types" - workers: 2 - reruns: 2 - - name: "other-2" - # enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines - path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils" - workers: 2 - reruns: 2 - - name: "other-3" - # remaining dirs ≈ 8.0k lines - path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores" - workers: 2 - reruns: 2 - - name: "root" - path: "tests/test_litellm/test_*.py" - workers: 2 - reruns: 2 - # tests/proxy_unit_tests split alphabetically (~48 files total) - - name: "proxy-unit-a1" - # test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest - path: "tests/proxy_unit_tests/test_[a-j]*.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-a2" - # test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback - path: "tests/proxy_unit_tests/test_[k-o]*.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-b1" - # lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*) - path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-b2" - # proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests - path: "tests/proxy_unit_tests/test_proxy_server.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-b3" - # proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests - path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-b4" - # proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter - path: "tests/proxy_unit_tests/test_proxy_utils.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-b5" - # proxy_token_counter (1279) - runs independently from utils - path: "tests/proxy_unit_tests/test_proxy_token_counter.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-b6" - # test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62) - path: "tests/proxy_unit_tests/test_[r-t]*.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-b7" - # test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157) - path: "tests/proxy_unit_tests/test_[u-z]*.py" - workers: 2 - reruns: 1 - - name: test (${{ matrix.test-group.name }}) - - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Install Poetry - run: pip install 'poetry==2.3.2' - - - name: Cache Poetry dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 - with: - path: | - ~/.cache/pypoetry - ~/.cache/pip - .venv - key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} - restore-keys: | - ${{ runner.os }}-poetry- - - - name: Install dependencies - run: | - poetry config virtualenvs.in-project true - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies - poetry run pip install google-genai==1.22.0 \ - google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - - - name: Setup litellm-enterprise - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ - - - name: Generate Prisma client - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache - run: | - poetry run pip install nodejs-wheel-binaries==24.13.1 - poetry run prisma generate --schema litellm/proxy/schema.prisma - - - name: Run tests - ${{ matrix.test-group.name }} - run: | - poetry run pytest ${{ matrix.test-group.path }} \ - --tb=short -vv \ - --maxfail=10 \ - -n ${{ matrix.test-group.workers }} \ - --reruns ${{ matrix.test-group.reruns }} \ - --reruns-delay 1 \ - --dist=loadscope \ - --durations=20 \ - --cov=litellm \ - --cov-report=xml:coverage-${{ matrix.test-group.name }}.xml \ - --cov-config=pyproject.toml - - - name: Save coverage report - if: always() - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - name: coverage-${{ matrix.test-group.name }} - path: coverage-${{ matrix.test-group.name }}.xml - retention-days: 1 - - upload-coverage: - name: Upload coverage to Codecov - needs: test - if: always() - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write # Required for OIDC tokenless upload - pull-requests: write # Required for Codecov PR comments - - steps: - - name: Checkout code - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - - - name: Download all coverage reports - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 - with: - pattern: coverage-* - path: coverage-reports - merge-multiple: true - - - name: Upload to Codecov - uses: codecov/codecov-action@aa56896cf108bd10b5eb883cd1d24196da57f695 # v5.5.4 - with: - use_oidc: true - directory: coverage-reports - root_dir: ${{ github.workspace }} - fail_ci_if_error: false diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index 0c040b3ebe..938647f5d0 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -31,23 +31,15 @@ jobs: with: python-version: "3.12" - - name: Install Poetry - run: pip install 'poetry==2.3.2' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - name: Install dependencies run: | - poetry lock - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install "pytest-retry==1.6.3" - poetry run pip install 'pytest-xdist==3.8.0' - poetry run pip install "google-genai==1.22.0" - poetry run pip install "google-cloud-aiplatform==1.115.0" - poetry run pip install "fastapi-offline==1.7.3" - poetry run pip install "python-multipart==0.0.22" - poetry run pip install "openapi-core==0.23.0" - - name: Setup litellm-enterprise as local package - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + uv lock --check + uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router - name: Run tests run: | - poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 + uv run --no-sync pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml index 1b228ab76b..313043e12f 100644 --- a/.github/workflows/test-mcp.yml +++ b/.github/workflows/test-mcp.yml @@ -2,7 +2,11 @@ name: LiteLLM MCP Tests (folder - tests/mcp_tests) on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" permissions: contents: read @@ -27,26 +31,16 @@ jobs: with: python-version: "3.12" - - name: Install Poetry - run: pip install 'poetry==2.3.2' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - name: Install dependencies run: | - poetry lock - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install "pytest==7.3.1" - poetry run pip install "pytest-retry==1.6.3" - poetry run pip install "pytest-cov==5.0.0" - poetry run pip install "pytest-asyncio==0.21.1" - poetry run pip install "respx==0.22.0" - poetry run pip install "pydantic==2.11.0" - poetry run pip install "mcp==1.25.0" - poetry run pip install 'pytest-xdist==3.8.0' - - - name: Setup litellm-enterprise as local package - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + uv lock --check + uv sync --frozen --group proxy-dev --extra proxy --extra semantic-router - name: Run MCP tests run: | - poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5 + uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5 diff --git a/.github/workflows/test-model-map.yaml b/.github/workflows/test-model-map.yaml index 429f9e1ce0..49821fca3a 100644 --- a/.github/workflows/test-model-map.yaml +++ b/.github/workflows/test-model-map.yaml @@ -2,7 +2,11 @@ name: Validate model_prices_and_context_window.json on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" permissions: contents: read diff --git a/.github/workflows/test-proxy-e2e-azure-batches.yml b/.github/workflows/test-proxy-e2e-azure-batches.yml deleted file mode 100644 index 7cbbe0b338..0000000000 --- a/.github/workflows/test-proxy-e2e-azure-batches.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: Proxy E2E Azure Batches Tests - -on: - pull_request: - branches: [main] - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - proxy_e2e_azure_batches_tests: - runs-on: ubuntu-latest - timeout-minutes: 30 - - services: - postgres: - image: postgres:15 - env: - POSTGRES_USER: llmproxy - POSTGRES_PASSWORD: dbpassword9090 - POSTGRES_DB: litellm - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Install Poetry - run: pip install 'poetry==2.3.2' - - - name: Cache Poetry dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 - with: - path: | - ~/.cache/pypoetry - ~/.cache/pip - .venv - key: ${{ runner.os }}-poetry-e2e-batches-${{ hashFiles('poetry.lock') }} - restore-keys: | - ${{ runner.os }}-poetry-e2e-batches- - ${{ runner.os }}-poetry- - - - name: Install dependencies - run: | - poetry config virtualenvs.in-project true - poetry install --with dev,proxy-dev --extras "proxy" - poetry run pip install psycopg2-binary==2.9.11 uvicorn==0.42.0 fastapi==0.135.2 httpx==0.28.1 tenacity==9.1.4 - - - name: Setup litellm-enterprise - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ - - - name: Generate Prisma client - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache - run: | - poetry run pip install nodejs-wheel-binaries==24.13.1 - poetry run prisma generate --schema litellm/proxy/schema.prisma - - - name: Run Prisma migrations - env: - DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm - run: | - cd litellm/proxy - poetry run prisma migrate deploy --schema schema.prisma - cd ../.. - - - name: Run Azure Batch E2E Tests - env: - DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm - USE_LOCAL_LITELLM: "true" - USE_MOCK_MODELS: "true" - USE_STATE_TRACKER: "true" - LITELLM_LOG: DEBUG - run: | - poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \ - -vv -s -k "test_e2e_managed_batch" \ - --tb=short \ - --maxfail=3 \ - --durations=10 diff --git a/.github/workflows/test-unit-core-utils.yml b/.github/workflows/test-unit-core-utils.yml index 2f3698fdf6..da1267756c 100644 --- a/.github/workflows/test-unit-core-utils.yml +++ b/.github/workflows/test-unit-core-utils.yml @@ -2,10 +2,16 @@ name: "Unit Tests: Core Utilities" on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -18,3 +24,4 @@ jobs: test-path: "tests/test_litellm/litellm_core_utils" workers: 2 reruns: 1 + artifact-name: core-utils diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml index d8b30de684..b2a8640223 100644 --- a/.github/workflows/test-unit-documentation.yml +++ b/.github/workflows/test-unit-documentation.yml @@ -2,7 +2,11 @@ name: "Unit Tests: Documentation Validation" on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" permissions: contents: read @@ -21,47 +25,47 @@ jobs: with: persist-credentials: false + - name: Checkout litellm-docs into docs/my-website (for documentation_tests) + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + repository: BerriAI/litellm-docs + path: docs/my-website + persist-credentials: false + - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - - name: Install Poetry - run: pip install 'poetry==2.3.2' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - - name: Cache Poetry dependencies + - name: Cache uv dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | - ~/.cache/pypoetry - ~/.cache/pip + ~/.cache/uv .venv - key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-poetry- + ${{ runner.os }}-uv- - name: Install dependencies run: | - poetry config virtualenvs.in-project true - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install google-genai==1.22.0 \ - google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - - - name: Setup litellm-enterprise - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router - name: Generate Prisma client env: PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | - poetry run pip install nodejs-wheel-binaries==24.13.1 - poetry run prisma generate --schema litellm/proxy/schema.prisma + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma # Run the same documentation tests that CircleCI ran (as direct Python scripts) - name: Run documentation validation tests run: | - poetry run python ./tests/documentation_tests/test_env_keys.py - poetry run python ./tests/documentation_tests/test_router_settings.py - poetry run python ./tests/documentation_tests/test_api_docs.py - poetry run python ./tests/documentation_tests/test_circular_imports.py + uv run --no-sync python ./tests/documentation_tests/test_env_keys.py + uv run --no-sync python ./tests/documentation_tests/test_router_settings.py + uv run --no-sync python ./tests/documentation_tests/test_api_docs.py + uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py diff --git a/.github/workflows/test-unit-enterprise-routing.yml b/.github/workflows/test-unit-enterprise-routing.yml index 13ae3efedb..ffc09dd8f9 100644 --- a/.github/workflows/test-unit-enterprise-routing.yml +++ b/.github/workflows/test-unit-enterprise-routing.yml @@ -2,10 +2,16 @@ name: "Unit Tests: Enterprise, Google GenAI & Routing" on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -22,3 +28,4 @@ jobs: tests/test_litellm/router_strategy workers: 2 reruns: 2 + artifact-name: enterprise-routing diff --git a/.github/workflows/test-unit-integrations.yml b/.github/workflows/test-unit-integrations.yml index 2789f99d81..b316ad5dfd 100644 --- a/.github/workflows/test-unit-integrations.yml +++ b/.github/workflows/test-unit-integrations.yml @@ -2,10 +2,16 @@ name: "Unit Tests: Integrations (Callbacks & Logging)" on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -18,3 +24,4 @@ jobs: test-path: "tests/test_litellm/integrations" workers: 2 reruns: 3 + artifact-name: integrations diff --git a/.github/workflows/test-unit-llm-providers.yml b/.github/workflows/test-unit-llm-providers.yml index 6c00272b0c..2a1912ce92 100644 --- a/.github/workflows/test-unit-llm-providers.yml +++ b/.github/workflows/test-unit-llm-providers.yml @@ -2,7 +2,11 @@ name: "Unit Tests: LLM Provider Transformations" on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" permissions: contents: read @@ -14,16 +18,26 @@ concurrency: jobs: vertex-ai: name: Vertex AI + permissions: + contents: read + id-token: write + pull-requests: write uses: ./.github/workflows/_test-unit-base.yml with: test-path: "tests/test_litellm/llms/vertex_ai" workers: 1 reruns: 2 + artifact-name: llm-vertex-ai other-providers: name: All Other Providers + permissions: + contents: read + id-token: write + pull-requests: write uses: ./.github/workflows/_test-unit-base.yml with: test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" workers: 2 reruns: 2 + artifact-name: llm-other-providers diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml index 9228decd7c..9add77ff42 100644 --- a/.github/workflows/test-unit-misc.yml +++ b/.github/workflows/test-unit-misc.yml @@ -2,10 +2,16 @@ name: "Unit Tests: MCP, Secrets, Containers & Misc" on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -29,3 +35,4 @@ jobs: tests/test_litellm/test_*.py workers: 2 reruns: 2 + artifact-name: misc diff --git a/.github/workflows/test-unit-proxy-auth.yml b/.github/workflows/test-unit-proxy-auth.yml index e71821db70..99882066a8 100644 --- a/.github/workflows/test-unit-proxy-auth.yml +++ b/.github/workflows/test-unit-proxy-auth.yml @@ -2,10 +2,16 @@ name: "Unit Tests: Proxy Auth & Key Management" on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -18,3 +24,4 @@ jobs: test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client" workers: 2 reruns: 2 + artifact-name: proxy-auth diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index bdfb6efeef..d5781f767f 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -3,7 +3,7 @@ name: "Unit Tests: Proxy DB Operations" # Uses DATABASE_URL secret — only runs on trusted branches, not PRs. on: push: - branches: [main, "litellm_*"] + branches: [main, "litellm_**"] permissions: contents: read @@ -12,34 +12,227 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +# Semantic matrix: each shard groups tests by concern (auth, server, logging, …) +# rather than alphabetical letter ranges. Adding a new test file means adding it +# to whichever group it belongs to, not reshuffling slices. +# +# Design targets: +# * Every shard runs in <= 7 minutes of wall-clock on the default runner. +# Most of a shard's time is pytest plugin load + xdist worker imports + +# pytest-cov instrumentation, not the tests themselves. Keeping per-shard +# work low and matching worker count to runner cores is what controls it. +# * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores +# oversubscribes 2x and workers fight for CPU during their cold-start +# imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective). +# * test_key_generate_prisma.py stays serial (workers=0) — it has event-loop +# conflicts with the logging worker when run in parallel. +# * test_proxy_utils.py runs as a single shard with --dist=worksteal so +# xdist balances its 188 parametrized cases across workers instead of +# pinning the whole file to one worker (the default --dist=loadscope +# behavior for single-file targets). +# * test_db_schema_migration.py is isolated because one test in it +# (test_aaaasschema_migration_check) takes ~170s — by itself it +# determines the shard's wall-clock floor. jobs: + # Fast guard — fails the workflow if a test_*.py file under + # tests/proxy_unit_tests/ is not referenced by any matrix entry below. + # The semantic-shard design (no catch-all "remaining" bucket) relies on + # every test file being explicitly assigned; this guard prevents a new + # file from silently dropping out of CI. + assert-shard-coverage: + runs-on: ubuntu-latest + timeout-minutes: 2 + permissions: + contents: read + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + - name: Assert every test_*.py is in a matrix shard + run: | + python3 - <<'PY' + import pathlib, sys, yaml + wf = yaml.safe_load(open(".github/workflows/test-unit-proxy-db.yml")) + matrix = wf["jobs"]["proxy-db"]["strategy"]["matrix"]["include"] + referenced = set() + for entry in matrix: + for token in entry["test-path"].split(): + if token.startswith("tests/proxy_unit_tests/"): + referenced.add(pathlib.PurePosixPath(token).name) + actual = {p.name for p in pathlib.Path("tests/proxy_unit_tests").iterdir() + if p.name.startswith("test_") and (p.suffix == ".py" or p.is_dir()) + and p.name != "test_configs"} + orphans = sorted(actual - referenced) + if orphans: + print("ERROR: the following files/dirs under tests/proxy_unit_tests/") + print(" are not assigned to any shard in test-unit-proxy-db.yml:") + for o in orphans: + print(f" - {o}") + print() + print("Add each to whichever semantic shard it belongs to.") + sys.exit(1) + print(f"OK: all {len(actual)} files assigned to a shard.") + PY + proxy-db: + needs: assert-shard-coverage + # Display only the semantic shard name in the checks UI instead of GHA's + # default "proxy-db (key-generation, tests/proxy_unit_tests/…, 0, loadscope, 20)" + # which includes every matrix field and gets truncated past the test-path. + name: ${{ matrix.test-group }} + permissions: + contents: read + id-token: write + pull-requests: write strategy: fail-fast: false matrix: include: - # Key generation tests must NOT run in parallel (event loop conflicts with logging worker) + # Must run serially — event-loop conflict with the logging worker. - test-group: key-generation test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py" workers: 0 - timeout: 30 + dist: loadscope + timeout: 20 + + # ---- auth: split into 2 shards ---- - test-group: auth-checks - test-path: "tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py" - workers: 8 - timeout: 20 - - test-group: remaining - test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py" - workers: 8 - timeout: 20 + test-path: >- + tests/proxy_unit_tests/test_auth_checks.py + tests/proxy_unit_tests/test_user_api_key_auth.py + workers: 4 + dist: loadscope + timeout: 15 + - test-group: jwt-and-keys + test-path: >- + tests/proxy_unit_tests/test_jwt.py + tests/proxy_unit_tests/test_jwt_key_mapping.py + tests/proxy_unit_tests/test_proxy_custom_auth.py + tests/proxy_unit_tests/test_key_generate_dynamodb.py + tests/proxy_unit_tests/test_deployed_proxy_keygen.py + workers: 4 + dist: loadscope + timeout: 15 + + # ---- test_proxy_utils.py, single shard, worksteal distribution ---- + - test-group: proxy-utils + test-path: "tests/proxy_unit_tests/test_proxy_utils.py" + workers: 4 + dist: worksteal + timeout: 15 + + # ---- proxy server: split into 2 shards ---- + - test-group: proxy-server-core + test-path: >- + tests/proxy_unit_tests/test_proxy_server.py + tests/proxy_unit_tests/test_proxy_server_keys.py + tests/proxy_unit_tests/test_proxy_server_caching.py + tests/proxy_unit_tests/test_proxy_server_langfuse.py + tests/proxy_unit_tests/test_proxy_server_spend.py + tests/proxy_unit_tests/test_aproxy_startup.py + workers: 4 + dist: loadscope + timeout: 15 + - test-group: proxy-runtime + test-path: >- + tests/proxy_unit_tests/test_proxy_config_unit_test.py + tests/proxy_unit_tests/test_proxy_routes.py + tests/proxy_unit_tests/test_proxy_gunicorn.py + tests/proxy_unit_tests/test_server_root_path.py + tests/proxy_unit_tests/test_proxy_pass_user_config.py + tests/proxy_unit_tests/test_proxy_token_counter.py + workers: 4 + dist: loadscope + timeout: 15 + + # ---- logging: split into 2 shards ---- + - test-group: custom-logging + test-path: >- + tests/proxy_unit_tests/test_custom_callback_input.py + tests/proxy_unit_tests/test_custom_logger_s3_gcs.py + tests/proxy_unit_tests/test_proxy_custom_logger.py + workers: 4 + dist: loadscope + timeout: 15 + - test-group: logging-misc + test-path: >- + tests/proxy_unit_tests/test_proxy_reject_logging.py + tests/proxy_unit_tests/test_audit_logs_proxy.py + tests/proxy_unit_tests/test_search_api_logging.py + workers: 4 + dist: loadscope + timeout: 15 + + # ---- db-and-spend: isolate the 170s schema-migration test ---- + # test_db_schema_migration.py has exactly one test, and that test + # is mostly waiting on `prisma migrate deploy` / `prisma migrate + # diff` subprocesses (~170s). It does no CPU-bound Python work + # inside the test. Running with workers=0 (serial, no xdist) + # skips the 4-worker cold-start cost we'd otherwise pay for a + # single test, saving ~4 minutes of wall-clock. + - test-group: schema-migration + test-path: "tests/proxy_unit_tests/test_db_schema_migration.py" + workers: 0 + dist: loadscope + timeout: 15 + - test-group: db-and-spend + test-path: >- + tests/proxy_unit_tests/test_prisma_client_backoff_retry.py + tests/proxy_unit_tests/test_db_schema_changes.py + tests/proxy_unit_tests/test_e2e_pod_lock_manager.py + tests/proxy_unit_tests/test_skills_db.py + tests/proxy_unit_tests/test_update_daily_tag_spend.py + tests/proxy_unit_tests/test_update_spend.py + tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py + workers: 4 + dist: loadscope + timeout: 15 + + # ---- guardrails + budget + hooks: split into 2 ---- + - test-group: guardrails-hooks + test-path: >- + tests/proxy_unit_tests/test_proxy_setting_guardrails.py + tests/proxy_unit_tests/test_banned_keyword_list.py + tests/proxy_unit_tests/test_unit_test_proxy_hooks.py + workers: 4 + dist: loadscope + timeout: 15 + - test-group: budgets + test-path: >- + tests/proxy_unit_tests/test_default_end_user_budget_simple.py + tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py + tests/proxy_unit_tests/test_zero_cost_model_budget_bypass.py + workers: 4 + dist: loadscope + timeout: 15 + + - test-group: endpoints-and-responses + test-path: >- + tests/proxy_unit_tests/test_blog_posts_endpoint.py + tests/proxy_unit_tests/test_models_fallback_endpoint.py + tests/proxy_unit_tests/test_google_endpoint_routing.py + tests/proxy_unit_tests/test_google_gemini_proxy_request.py + tests/proxy_unit_tests/test_get_favicon.py + tests/proxy_unit_tests/test_get_image.py + tests/proxy_unit_tests/test_ui_path_detection.py + tests/proxy_unit_tests/test_prompt_test_endpoint.py + tests/proxy_unit_tests/test_check_batch_cost.py + tests/proxy_unit_tests/test_check_responses_cost.py + tests/proxy_unit_tests/test_response_polling_handler.py + tests/proxy_unit_tests/test_response_polling_pre_call_checks.py + tests/proxy_unit_tests/test_realtime_cache.py + tests/proxy_unit_tests/test_proxy_exception_mapping.py + tests/proxy_unit_tests/test_custom_tokenizer_bug.py + tests/proxy_unit_tests/test_model_response_typing + workers: 4 + dist: loadscope + timeout: 15 uses: ./.github/workflows/_test-unit-services-base.yml with: test-path: ${{ matrix.test-path }} workers: ${{ matrix.workers }} reruns: 2 timeout-minutes: ${{ matrix.timeout }} - enable-redis: false enable-postgres: true - secrets: - DATABASE_URL: ${{ secrets.DATABASE_URL }} - POSTGRES_USER: ${{ secrets.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + dist: ${{ matrix.dist }} + artifact-name: proxy-db-${{ matrix.test-group }} diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml index caff3b3ae0..1439b2c07f 100644 --- a/.github/workflows/test-unit-proxy-endpoints.yml +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -2,10 +2,16 @@ name: "Unit Tests: Proxy API Endpoints" on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -30,6 +36,9 @@ jobs: tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/prompts + tests/test_litellm/proxy/rag_endpoints + tests/test_litellm/proxy/realtime_endpoints tests/test_litellm/proxy/ui_crud_endpoints workers: 2 reruns: 2 + artifact-name: proxy-endpoints diff --git a/.github/workflows/test-unit-proxy-infra.yml b/.github/workflows/test-unit-proxy-infra.yml index 4dfbbe317e..336e53ee3d 100644 --- a/.github/workflows/test-unit-proxy-infra.yml +++ b/.github/workflows/test-unit-proxy-infra.yml @@ -2,10 +2,16 @@ name: "Unit Tests: Proxy Infrastructure" on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -26,3 +32,4 @@ jobs: tests/test_litellm/proxy/test_*.py workers: 2 reruns: 2 + artifact-name: proxy-infra diff --git a/.github/workflows/test-unit-proxy-legacy.yml b/.github/workflows/test-unit-proxy-legacy.yml index a939113726..5768551f9b 100644 --- a/.github/workflows/test-unit-proxy-legacy.yml +++ b/.github/workflows/test-unit-proxy-legacy.yml @@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Legacy Tests" on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" permissions: contents: read @@ -24,7 +28,7 @@ jobs: - name: "key-generation" path: "tests/proxy_unit_tests/test_[k-o]*.py" - name: "proxy-config" - path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py" + path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py" - name: "proxy-server" path: "tests/proxy_unit_tests/test_proxy_server.py" - name: "proxy-server-extras" @@ -50,43 +54,36 @@ jobs: with: python-version: "3.12" - - name: Install Poetry - run: pip install 'poetry==2.3.2' + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" - - name: Cache Poetry dependencies + - name: Cache uv dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | - ~/.cache/pypoetry - ~/.cache/pip + ~/.cache/uv .venv - key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-poetry- + ${{ runner.os }}-uv- - name: Install dependencies run: | - poetry config virtualenvs.in-project true - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install google-genai==1.22.0 \ - google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - - - name: Setup litellm-enterprise - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router - name: Generate Prisma client env: PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | - poetry run pip install nodejs-wheel-binaries==24.13.1 - poetry run prisma generate --schema litellm/proxy/schema.prisma + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Run tests - ${{ matrix.test-group.name }} env: TEST_PATH: ${{ matrix.test-group.path }} run: | - poetry run pytest ${TEST_PATH} \ + uv run --no-sync pytest ${TEST_PATH} \ --tb=short -vv \ --maxfail=10 \ -n 2 \ diff --git a/.github/workflows/test-unit-responses-caching-types.yml b/.github/workflows/test-unit-responses-caching-types.yml index 7f3acac280..13069be9e3 100644 --- a/.github/workflows/test-unit-responses-caching-types.yml +++ b/.github/workflows/test-unit-responses-caching-types.yml @@ -2,10 +2,16 @@ name: "Unit Tests: Responses, Caching & Types" on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -18,3 +24,4 @@ jobs: test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types" workers: 2 reruns: 2 + artifact-name: responses-caching-types diff --git a/.github/workflows/test-unit-security.yml b/.github/workflows/test-unit-security.yml index b38c82b1c2..4ee8989702 100644 --- a/.github/workflows/test-unit-security.yml +++ b/.github/workflows/test-unit-security.yml @@ -1,12 +1,16 @@ name: "Unit Tests: Security" -# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. +# Kept push-only (was previously required by DATABASE_URL secret scoping; +# now the postgres credentials are ephemeral localhost values but the +# push-trigger stays to match the proxy-db workflow cadence). on: push: - branches: [main, "litellm_*"] + branches: [main, "litellm_**"] permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -20,9 +24,5 @@ jobs: workers: 1 reruns: 2 timeout-minutes: 20 - enable-redis: false enable-postgres: true - secrets: - DATABASE_URL: ${{ secrets.DATABASE_URL }} - POSTGRES_USER: ${{ secrets.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + artifact-name: security diff --git a/.github/workflows/test_server_root_path.yml b/.github/workflows/test_server_root_path.yml index 47636ce8e9..155445acdf 100644 --- a/.github/workflows/test_server_root_path.yml +++ b/.github/workflows/test_server_root_path.yml @@ -4,12 +4,16 @@ permissions: on: pull_request: - branches: [main] + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" jobs: test-server-root-path: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 30 strategy: matrix: @@ -21,6 +25,12 @@ jobs: with: persist-credentials: false + - name: Free up disk space + run: | + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/share/boost + sudo apt-get clean + df -h / + - name: Set up Docker Buildx uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12 diff --git a/scripts/install.sh b/scripts/install.sh index 03ae31cd18..c28d7da872 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -11,6 +11,7 @@ MIN_PYTHON_MINOR=9 # NOTE: before merging, this must stay as "litellm[proxy]" to install from PyPI. LITELLM_PACKAGE="litellm[proxy]" +UV_VERSION="0.10.9" # ── colours ──────────────────────────────────────────────────────────────── if [ -t 1 ]; then @@ -69,13 +70,34 @@ if [ -z "$PYTHON_BIN" ]; then die "Python ${MIN_PYTHON_MAJOR}.${MIN_PYTHON_MINOR}+ is required but not found. Install it from https://python.org/downloads or via your package manager: macOS: brew install python@3 - Ubuntu: sudo apt install python3 python3-pip" + Ubuntu: sudo apt install python3" fi -# ── pip detection ────────────────────────────────────────────────────────── -if ! "$PYTHON_BIN" -m pip --version >/dev/null 2>&1; then - die "pip is not available. Install it with: - $PYTHON_BIN -m ensurepip --upgrade" +# ── uv detection / install ──────────────────────────────────────────────── +UV_BIN="" +CURRENT_UV_VERSION="" +for candidate in uv "$HOME/.local/bin/uv"; do + if command -v "$candidate" >/dev/null 2>&1; then + UV_BIN="$(command -v "$candidate")" + break + elif [ -x "$candidate" ]; then + UV_BIN="$candidate" + break + fi +done + +if [ -n "$UV_BIN" ]; then + CURRENT_UV_VERSION="$("$UV_BIN" --version 2>/dev/null | awk '{print $2}' | head -1 || true)" +fi + +if [ -z "$UV_BIN" ] || [ "${CURRENT_UV_VERSION:-}" != "$UV_VERSION" ]; then + header "Installing uv…" + if [ -n "${CURRENT_UV_VERSION:-}" ]; then + info "Upgrading uv from ${CURRENT_UV_VERSION} to ${UV_VERSION}" + fi + curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" | env UV_NO_MODIFY_PATH=1 sh \ + || die "uv installation failed. Try manually: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh" + UV_BIN="$HOME/.local/bin/uv" fi # ── install ──────────────────────────────────────────────────────────────── @@ -83,23 +105,15 @@ echo "" header "Installing litellm[proxy]…" echo "" -"$PYTHON_BIN" -m pip install --only-binary :all: --upgrade "${LITELLM_PACKAGE}" \ - || die "pip install failed. Try manually: $PYTHON_BIN -m pip install --only-binary :all: '${LITELLM_PACKAGE}'" +"$UV_BIN" tool install --python "$PYTHON_BIN" --force "${LITELLM_PACKAGE}" \ + || die "uv tool install failed. Try manually: $UV_BIN tool install --python '$PYTHON_BIN' '${LITELLM_PACKAGE}'" -# ── find the litellm binary installed by pip for this Python ─────────────── -# sysconfig.get_path('scripts') is where pip puts console scripts — reliable -# even when the Python lives in a libexec/ symlink tree (e.g. Homebrew). -SCRIPTS_DIR="$("$PYTHON_BIN" -c 'import sysconfig; print(sysconfig.get_path("scripts"))')" +# ── find the litellm binary installed by uv tool ─────────────────────────── +SCRIPTS_DIR="$("$UV_BIN" tool dir --bin)" LITELLM_BIN="${SCRIPTS_DIR}/litellm" if [ ! -x "$LITELLM_BIN" ]; then - # Fall back to user-base bin (pip install --user) - USER_BIN="$("$PYTHON_BIN" -c 'import site; print(site.getuserbase())')/bin" - LITELLM_BIN="${USER_BIN}/litellm" -fi - -if [ ! -x "$LITELLM_BIN" ]; then - die "litellm binary not found after install. Try: $PYTHON_BIN -m pip install --user '${LITELLM_PACKAGE}'" + die "litellm binary not found after install. Try: $UV_BIN tool install --python '$PYTHON_BIN' '${LITELLM_PACKAGE}'" fi # ── success banner ─────────────────────────────────────────────────────────