From 2875fe8e4900b63955221cbf5d68f0413af823c6 Mon Sep 17 00:00:00 2001 From: jquinter Date: Fri, 6 Feb 2026 12:34:31 -0300 Subject: [PATCH] ci: add matrix-based parallel test workflow (#19942) Split tests/test_litellm into 10 parallel CI jobs using GitHub Actions matrix strategy to reduce PR feedback time from ~25 min to ~8-10 min. Changes: - Add new test-litellm-matrix.yml workflow with 10 matrix jobs: - llms (~225 files, 4 workers) - proxy-guardrails (~51 files, 4 workers) - proxy-core (~52 files, 4 workers) - proxy-misc (~77 files, 4 workers) - integrations (~60 files, 4 workers) - core-utils (~32 files, 2 workers) - other (~69 files, 4 workers) - includes all previously uncovered dirs - root (~34 files, 4 workers) - proxy-unit-a (~20 files, 2 workers) - proxy-unit-b (~28 files, 2 workers) - Deprecate test-litellm.yml (moved to workflow_dispatch for manual use) - Add matching Makefile targets for local testing: - make test-unit-llms - make test-unit-proxy-guardrails - make test-unit-proxy-core - make test-unit-proxy-misc - make test-unit-integrations - make test-unit-core-utils - make test-unit-other - make test-unit-root - make test-proxy-unit-a - make test-proxy-unit-b Benefits: - ~3x faster wall-clock time through parallelization - Dependency caching for faster subsequent runs - Concurrency control to cancel stale runs - Better failure isolation per test group Co-authored-by: Claude Opus 4.5 --- .github/workflows/test-litellm-matrix.yml | 109 ++++++++++++++++++++++ .github/workflows/test-litellm.yml | 8 +- Makefile | 46 ++++++++- 3 files changed, 160 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/test-litellm-matrix.yml diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml new file mode 100644 index 0000000000..b442c7dd5f --- /dev/null +++ b/.github/workflows/test-litellm-matrix.yml @@ -0,0 +1,109 @@ +name: LiteLLM Unit Tests (Matrix) + +on: + pull_request: + branches: [main] + +# Cancel in-progress runs for the same PR +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + test-group: + # tests/test_litellm split by subdirectory (~560 files total) + - name: "llms" + path: "tests/test_litellm/llms" + workers: 4 + # tests/test_litellm/proxy split by subdirectory (~180 files total) + - name: "proxy-guardrails" + path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers" + workers: 4 + - name: "proxy-core" + path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine" + workers: 4 + - name: "proxy-misc" + path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py" + workers: 4 + - name: "integrations" + path: "tests/test_litellm/integrations" + workers: 4 + - name: "core-utils" + path: "tests/test_litellm/litellm_core_utils" + workers: 2 + - name: "other" + path: "tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types" + workers: 4 + - name: "root" + path: "tests/test_litellm/test_*.py" + workers: 4 + # tests/proxy_unit_tests split alphabetically (~48 files total) + - name: "proxy-unit-a" + path: "tests/proxy_unit_tests/test_[a-o]*.py" + workers: 2 + - name: "proxy-unit-b" + path: "tests/proxy_unit_tests/test_[p-z]*.py" + workers: 2 + + name: test (${{ matrix.test-group.name }}) + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Poetry + uses: snok/install-poetry@v1 + + - name: Cache Poetry dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install pytest-retry==1.6.3 pytest-xdist google-genai==1.22.0 \ + google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core + + - name: Setup litellm-enterprise + run: | + cd enterprise && poetry run pip install -e . && cd .. + + - name: Run tests - ${{ matrix.test-group.name }} + run: | + poetry run pytest ${{ matrix.test-group.path }} \ + --tb=short -vv \ + --maxfail=10 \ + -n ${{ matrix.test-group.workers }} \ + --durations=20 + + # Aggregate job to require all matrix jobs pass + test-complete: + needs: test + runs-on: ubuntu-latest + if: always() + steps: + - name: Check test results + run: | + if [ "${{ needs.test.result }}" != "success" ]; then + echo "Some test groups failed" + exit 1 + fi + echo "All test groups passed!" diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index d9cf2e74a1..dc9b48c28f 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -1,8 +1,12 @@ name: LiteLLM Mock Tests (folder - tests/test_litellm) +# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs +# the same tests in parallel across 10 jobs for faster CI times. +# Kept for manual debugging only. on: - pull_request: - branches: [ main ] + workflow_dispatch: # Manual trigger only + # pull_request: + # branches: [ main ] jobs: test: diff --git a/Makefile b/Makefile index b867d7ea35..74031f418d 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ # LiteLLM Makefile # Simple Makefile for running tests and basic development tasks -.PHONY: help test test-unit test-integration test-unit-helm \ +.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \ + test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \ + test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \ info lint lint-dev format \ install-dev install-proxy-dev install-test-deps \ install-helm-unittest check-circular-imports check-import-safety @@ -25,6 +27,16 @@ help: @echo " make check-import-safety - Check import safety" @echo " make test - Run all tests" @echo " make test-unit - Run unit tests (tests/test_litellm)" + @echo " make test-unit-llms - Run LLM provider tests (~225 files)" + @echo " make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)" + @echo " make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)" + @echo " make test-unit-proxy-misc - Run proxy misc tests (~77 files)" + @echo " make test-unit-integrations - Run integration tests (~60 files)" + @echo " make test-unit-core-utils - Run core utils tests (~32 files)" + @echo " make test-unit-other - Run other tests (caching, responses, etc., ~69 files)" + @echo " make test-unit-root - Run root-level tests (~34 files)" + @echo " make test-proxy-unit-a - Run proxy_unit_tests (a-o, ~20 files)" + @echo " make test-proxy-unit-b - Run proxy_unit_tests (p-z, ~28 files)" @echo " make test-integration - Run integration tests" @echo " make test-unit-helm - Run helm unit tests" @@ -129,6 +141,38 @@ test: test-unit: install-test-deps poetry run pytest tests/test_litellm -x -vv -n 4 +# Matrix test targets (matching CI workflow groups) +test-unit-llms: install-test-deps + poetry run pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20 + +test-unit-proxy-guardrails: install-test-deps + poetry run pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20 + +test-unit-proxy-core: install-test-deps + poetry run pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20 + +test-unit-proxy-misc: install-test-deps + poetry run pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20 + +test-unit-integrations: install-test-deps + poetry run pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20 + +test-unit-core-utils: install-test-deps + poetry run pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20 + +test-unit-other: install-test-deps + poetry run pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20 + +test-unit-root: install-test-deps + poetry run pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20 + +# Proxy unit tests (tests/proxy_unit_tests split alphabetically) +test-proxy-unit-a: install-test-deps + poetry run pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20 + +test-proxy-unit-b: install-test-deps + poetry run pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20 + test-integration: poetry run pytest tests/ -k "not test_litellm"