From 62dca9e9771b7e2c2c99a2f81afba3912b4533cf Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 16 May 2026 10:56:32 -0700 Subject: [PATCH] fix(ci): flag codecov uploads, enable carryforward, close coverage gaps (#28028) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ci): flag codecov uploads and enable carryforward Coverage uploads from GHA and CircleCI were unflagged. Commits that receive the push-triggered workflows more than once (re-runs, or branches cut at the same SHA) accumulated many overlapping flagless sessions, and Codecov's per-commit merge dropped the largest, ubiquitously-imported files (router.py, proxy_server.py, main.py, utils.py, cost_calculator.py) from the report even though the uploaded XMLs contained them. - codecov.yaml: flag_management.default_rules.carryforward: true - GHA reusable bases: tag each upload with its workflow/shard name - CircleCI: tag the combined upload "circleci"; also combine the agent / google_generate_content_endpoint / litellm_utils datafiles that were produced and required but missing from the combine list * fix(ci): close coverage gaps in proxy-legacy, router-unit, auth-ui, caching-redis - test-unit-proxy-legacy: route through _test-unit-base so the full proxy_unit_tests suite (incl. comprehensive test_proxy_server*.py) is measured and uploaded with per-group flags (was plain pytest, no --cov) - _test-unit-services-base: declare the enable-redis input + the six secrets test-unit-caching-redis passes; that workflow had a workflow_call signature mismatch and startup_failed on every push (never ran). Changes are additive/optional - proxy-db and security callers unchanged - circleci: add --cov + persist + combine + upload-coverage requires for litellm_router_unit_testing (tests/router_unit_tests) and auth_ui_unit_tests (tests/proxy_admin_ui_tests); neither was covered anywhere. Redundant -k subset jobs left as-is (local_testing covers them) * fix(ci): remove dead GHA Redis workflow; keep Redis on CircleCI only CircleCI redis_caching_unit_tests already runs the exact same files (tests/local_testing/test_dual_cache.py, test_redis_batch_optimizations.py, test_router_utils.py) with --cov, and that datafile is already combined and uploaded. The GHA test-unit-caching-redis workflow was redundant and had never run (workflow_call signature mismatch -> startup_failure on every push). - Delete .github/workflows/test-unit-caching-redis.yml - Revert _test-unit-services-base.yml to the flag-fix state (drop the enable-redis input / secrets / env wiring added only to prop up the GHA Redis workflow); the verified per-upload flags line is kept - The only single-star "litellm_*" branch glob lived in the deleted file; no other single-star globs exist, so none remain to widen * fix(ci): keep proxy-legacy as a standalone job to preserve required check names Routing proxy-legacy through the reusable workflow renamed each check from the bare matrix name (e.g. "proxy-response-and-misc") to "proxy-response-and-misc / Run tests". Those bare names are required status checks in branch protection, so the old contexts never reported and PRs sat "Expected — Waiting for status to be reported" indefinitely. Restore the original standalone matrix job (job name == matrix name, so the required contexts report again) and add coverage in place: --cov on pytest plus an OIDC Codecov upload flagged proxy-legacy-. Net effect of the gap-#2 fix is preserved (flagged coverage for tests/proxy_unit_tests/**) without changing any check name. * revert(ci): drop all proxy-legacy changes from this PR tests/proxy_unit_tests/** is already fully covered by test-unit-proxy-db (its shard-coverage guard fails CI if any file in that dir is unassigned), which this PR already flags + carryforwards. Adding --cov and id-token:write to the legacy pull_request job was redundant and put OIDC on a job that runs untrusted PR code. Restore the file to the base version verbatim so this PR no longer touches proxy-legacy at all (also restores its original required check names). Retiring proxy-legacy in favor of proxy-db on pull_request is a separate effort that needs a branch-protection change. --- .circleci/config.yml | 27 ++++++++++++- .github/workflows/_test-unit-base.yml | 1 + .../workflows/_test-unit-services-base.yml | 1 + .github/workflows/test-unit-caching-redis.yml | 38 ------------------- codecov.yaml | 10 +++++ 5 files changed, 38 insertions(+), 39 deletions(-) delete mode 100644 .github/workflows/test-unit-caching-redis.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a5b6da77f..32983c92e2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -409,14 +409,25 @@ jobs: --verbose \ --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ + --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 2" no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml auth_ui_unit_tests_coverage.xml + mv .coverage auth_ui_unit_tests_coverage # Store test results - store_test_results: path: test-results + - persist_to_workspace: + root: . + paths: + - auth_ui_unit_tests_coverage.xml + - auth_ui_unit_tests_coverage litellm_router_testing: # Runs all tests with the "router" keyword docker: @@ -493,13 +504,24 @@ jobs: --verbose \ --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ + --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml router_unit_tests_coverage.xml + mv .coverage router_unit_tests_coverage # Store test results - store_test_results: path: test-results + - persist_to_workspace: + root: . + paths: + - router_unit_tests_coverage.xml + - router_unit_tests_coverage litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword docker: - *python312_image @@ -2280,10 +2302,11 @@ jobs: - run: name: Combine Coverage command: | - uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage + uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage uv tool run --from 'coverage[toml]==7.10.6' coverage xml - codecov/upload: file: ./coverage.xml + flags: circleci ui_build: docker: @@ -2669,6 +2692,8 @@ workflows: - local_testing_part1 - local_testing_part2 - litellm_assistants_api_testing + - litellm_router_unit_testing + - auth_ui_unit_tests - db_migration_disable_update_check: requires: - build_docker_database_image diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 64862f3c8f..7e91341ac7 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -132,4 +132,5 @@ jobs: use_oidc: true directory: coverage-reports root_dir: ${{ github.workspace }} + flags: ${{ inputs.artifact-name }} fail_ci_if_error: false diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml index 9dd668cbf3..7f973d8caf 100644 --- a/.github/workflows/_test-unit-services-base.yml +++ b/.github/workflows/_test-unit-services-base.yml @@ -186,4 +186,5 @@ jobs: use_oidc: true directory: coverage-reports root_dir: ${{ github.workspace }} + flags: ${{ inputs.artifact-name }} fail_ci_if_error: false diff --git a/.github/workflows/test-unit-caching-redis.yml b/.github/workflows/test-unit-caching-redis.yml deleted file mode 100644 index ca274324f2..0000000000 --- a/.github/workflows/test-unit-caching-redis.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: "Unit Tests: Caching (Redis)" - -# Uses cloud Redis credentials — only runs on trusted branches, not PRs. -# This prevents external PRs from accessing Redis credentials. -on: - push: - branches: [main, "litellm_*"] - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - caching-redis: - uses: ./.github/workflows/_test-unit-services-base.yml - with: - # Redis-only tests that do NOT require provider API keys. - # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py, - # test_router_caching.py) are in Phase 3 integration workflows. - test-path: >- - tests/local_testing/test_dual_cache.py - tests/local_testing/test_redis_batch_optimizations.py - tests/local_testing/test_router_utils.py - workers: 2 - reruns: 2 - timeout-minutes: 20 - enable-redis: true - enable-postgres: false - secrets: - REDIS_HOST: ${{ secrets.REDIS_HOST }} - REDIS_PORT: ${{ secrets.REDIS_PORT }} - REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} - DATABASE_URL: ${{ secrets.DATABASE_URL }} - POSTGRES_USER: ${{ secrets.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/codecov.yaml b/codecov.yaml index 8609d3143d..58681b884d 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -3,6 +3,16 @@ codecov: notify: wait_for_ci: false # post as soon as expected uploads arrive, don't wait on CI +# Uploads are flagged per workflow/shard (GHA) or "circleci". carryforward makes +# a re-upload of a flag replace its prior session instead of accumulating a +# conflicting one, and lets a commit reuse a flag from its parent when that flag +# was not re-uploaded. Required because the same commit can receive the +# push-triggered workflows more than once (re-runs / branches cut at the same +# SHA); flagless overlapping sessions made Codecov drop the largest files. +flag_management: + default_rules: + carryforward: true + component_management: individual_components: - component_id: "Router"