From 1b74c35b89ae3e537654ca67cdeeafe53e7bd449 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 22 Apr 2026 12:11:48 -0700 Subject: [PATCH] [Infra] Move non-API-key CCI jobs to GitHub Actions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Principle: GHA handles work that doesn't need external API keys; CCI stays for integration tests that hit real API endpoints. Four CCI jobs moved to new or extended GHA workflows: 1. check_code_and_doc_quality (was 25 runs: ruff + import-safety + 21 code_coverage_tests + 3 documentation_tests + circular-imports). - The 21 tests/code_coverage_tests/*.py scripts and the 3 tests/documentation_tests/*.py scripts run in the new .github/workflows/test-code-quality.yml workflow. - ruff, import-safety, and circular-imports were already run by .github/workflows/test-linting.yml — no new migration needed. - The 3 documentation_tests scripts read docs/my-website/docs/proxy/config_settings.md. Since docs have moved to BerriAI/litellm-docs, the GHA workflow checks out that repo and symlinks docs/my-website -> the checkout so the existing hardcoded paths resolve without touching the scripts. The stale local docs/my-website/ copy in this repo will be removed in a separate PR. 2. semgrep (custom-rule SAST against .semgrep/rules). - New .github/workflows/test-semgrep.yml. 3. installing_litellm_on_python + installing_litellm_on_python_3_13 (pip install compat checks on Python 3.12 and 3.13). - New .github/workflows/test-install-litellm.yml as a matrix job. - 3.12 run also verifies litellm_enterprise import; 3.13 run skips that check (matches previous CCI behavior). - installing_litellm_on_python_v2_migration_resolver stays in CCI because it requires a postgres service. CCI .circleci/config.yml: -112 lines, 4 jobs and their workflow refs removed. --- .circleci/config.yml | 124 +++++---------------- .github/workflows/test-code-quality.yml | 136 ++++++++++++++++++++++++ .github/workflows/test-semgrep.yml | 39 +++++++ 3 files changed, 199 insertions(+), 100 deletions(-) create mode 100644 .github/workflows/test-code-quality.yml create mode 100644 .github/workflows/test-semgrep.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ea80be317..ba9d452000 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -164,25 +164,6 @@ jobs: command: | uv run --no-sync python -m pytest tests/windows_tests/test_litellm_on_windows.py -v - semgrep: - docker: - - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: medium - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Run Semgrep (custom rules only) - command: | - echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" - export PATH="$HOME/.local/bin:$PATH" - uv tool run --from 'semgrep==1.157.0' semgrep scan --config .semgrep/rules . --error - local_testing_part1: docker: - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c @@ -1283,6 +1264,30 @@ jobs: ls uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver" + installing_litellm_on_python_3_13: + docker: + - image: cimg/python:3.13.1@sha256:87b243ae80d154db75ce5e58af16c72c5dd4b1e23e5c7264a816e85e0c440c13 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: medium + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.13 + - run: + name: Run tests + command: | + pwd + ls + uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver" + installing_litellm_on_python_v2_migration_resolver: docker: - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c @@ -1316,29 +1321,6 @@ jobs: uv run --no-sync python -m pytest -vv \ tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver - installing_litellm_on_python_3_13: - docker: - - image: cimg/python:3.13.1@sha256:87b243ae80d154db75ce5e58af16c72c5dd4b1e23e5c7264a816e85e0c440c13 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: medium - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.13 - - run: - name: Run tests - command: | - pwd - ls - uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver" helm_chart_testing: machine: image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker @@ -1419,52 +1401,6 @@ jobs: kind delete cluster --name litellm-test when: always # This ensures cleanup runs even if previous steps fail - check_code_and_doc_quality: - docker: - - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project/litellm - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - run: uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) - - run: uv run --no-sync ruff check ./litellm - # - run: python ./tests/documentation_tests/test_general_setting_keys.py - - run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py - - run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py - - run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py - - run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py - - run: uv run --no-sync python ./tests/code_coverage_tests/info_log_check.py - - run: uv run --no-sync python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py - - run: uv run --no-sync python ./tests/code_coverage_tests/test_ban_set_verbose.py - - run: uv run --no-sync python ./tests/code_coverage_tests/code_qa_check_tests.py - - run: uv run --no-sync python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py - - run: uv run --no-sync python ./tests/code_coverage_tests/test_proxy_types_import.py - - run: uv run --no-sync python ./tests/code_coverage_tests/callback_manager_test.py - - run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py - - run: uv run --no-sync python ./tests/code_coverage_tests/test_router_strategy_async.py - - run: uv run --no-sync python ./tests/code_coverage_tests/litellm_logging_code_coverage.py - - run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py - - run: uv run --no-sync python ./tests/documentation_tests/test_router_settings.py - - run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py - - run: uv run --no-sync python ./tests/code_coverage_tests/ensure_async_clients_test.py - - run: uv run --no-sync python ./tests/code_coverage_tests/enforce_llms_folder_style.py - - run: uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py - - run: uv run --no-sync python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py - - run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py - - run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py - - run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py - - run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py - # helm lint is handled by the dedicated helm_chart_testing job - db_migration_disable_update_check: machine: image: ubuntu-2204:2024.04.1 @@ -2603,12 +2539,6 @@ workflows: only: - main - /litellm_.*/ - - semgrep: - filters: - branches: - only: - - main - - /litellm_.*/ - local_testing_part1: filters: branches: @@ -2645,12 +2575,6 @@ workflows: only: - main - /litellm_.*/ - - check_code_and_doc_quality: - filters: - branches: - only: - - main - - /litellm_.*/ - ui_build: filters: branches: diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml new file mode 100644 index 0000000000..da0cbcd915 --- /dev/null +++ b/.github/workflows/test-code-quality.yml @@ -0,0 +1,136 @@ +name: Code Quality Checks + +on: + pull_request: + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + code-quality: + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Checkout litellm-docs (for documentation_tests) + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + repository: BerriAI/litellm-docs + path: _litellm_docs_checkout + persist-credentials: false + + - name: Wire up docs path expected by documentation_tests/* + run: | + # documentation_tests scripts read from docs/my-website/docs/... + # In litellm-docs the same files live at docs/... (repo root). + # Point docs/my-website -> litellm-docs checkout so the paths resolve. + rm -rf docs/my-website + ln -s ../_litellm_docs_checkout docs/my-website + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + + - name: Cache uv dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/uv + .venv + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install dependencies + run: uv sync --frozen --all-groups --all-extras + + - name: check_licenses + run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py + + - name: check_provider_folders_documented + run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py + + - name: router_code_coverage + run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py + + - name: test_chat_completion_imports + run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py + + - name: info_log_check + run: uv run --no-sync python ./tests/code_coverage_tests/info_log_check.py + + - name: check_guardrail_apply_decorator + run: uv run --no-sync python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py + + - name: test_ban_set_verbose + run: uv run --no-sync python ./tests/code_coverage_tests/test_ban_set_verbose.py + + - name: code_qa_check_tests + run: uv run --no-sync python ./tests/code_coverage_tests/code_qa_check_tests.py + + - name: check_get_model_cost_key_performance + run: uv run --no-sync python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py + + - name: test_proxy_types_import + run: uv run --no-sync python ./tests/code_coverage_tests/test_proxy_types_import.py + + - name: callback_manager_test + run: uv run --no-sync python ./tests/code_coverage_tests/callback_manager_test.py + + - name: recursive_detector + run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py + + - name: test_router_strategy_async + run: uv run --no-sync python ./tests/code_coverage_tests/test_router_strategy_async.py + + - name: litellm_logging_code_coverage + run: uv run --no-sync python ./tests/code_coverage_tests/litellm_logging_code_coverage.py + + - name: ensure_async_clients_test + run: uv run --no-sync python ./tests/code_coverage_tests/ensure_async_clients_test.py + + - name: enforce_llms_folder_style + run: uv run --no-sync python ./tests/code_coverage_tests/enforce_llms_folder_style.py + + - name: prevent_key_leaks_in_exceptions + run: uv run --no-sync python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py + + - name: check_unsafe_enterprise_import + run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py + + - name: ban_copy_deepcopy_kwargs + run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py + + - name: check_fastuuid_usage + run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py + + - name: memory_test + run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py + + - name: documentation_test_env_keys + run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py + + - name: documentation_test_router_settings + run: uv run --no-sync python ./tests/documentation_tests/test_router_settings.py + + - name: documentation_test_api_docs + run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py diff --git a/.github/workflows/test-semgrep.yml b/.github/workflows/test-semgrep.yml new file mode 100644 index 0000000000..2ba23e44da --- /dev/null +++ b/.github/workflows/test-semgrep.yml @@ -0,0 +1,39 @@ +name: Semgrep + +on: + pull_request: + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + semgrep: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + + - name: Run Semgrep (custom rules) + run: uv tool run --from 'semgrep==1.157.0' semgrep scan --config .semgrep/rules . --error