diff --git a/.circleci/config.yml b/.circleci/config.yml index 12e3cb1f6b..d4943b59c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,7 +42,7 @@ commands: "pydantic==2.11.0" "mcp==1.25.0" "requests-mock>=1.12.1" \ "responses==0.25.7" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" \ "pytest-cov==5.0.0" "semantic_router==0.1.10" "fastapi-offline==1.7.3" \ - "a2a" + "a2a" "parameterized>=0.9.0" - setup_litellm_enterprise_pip - save_cache: paths: @@ -406,119 +406,6 @@ jobs: # Store test results - store_test_results: path: test-results - caching_unit_tests: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - resource_class: large - working_directory: ~/project - parallelism: 2 - - steps: - - checkout - - setup_google_dns - - run: - name: DNS lookup for Redis host - command: | - sudo apt-get update - sudo apt-get install -y dnsutils - dig redis-19899.c239.us-east-1-2.ec2.redns.redis-cloud.com +short - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - - restore_cache: - keys: - - v2-caching-deps-{{ checksum ".circleci/requirements.txt" }} - - v2-caching-deps- - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "websockets==13.1.0" - pip install "pytest-xdist==3.6.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - /home/circleci/.pyenv/versions - - /home/circleci/.local - key: v2-caching-deps-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - pwd - ls - mkdir -p test-results - - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") - - echo "$TEST_FILES" | circleci tests run \ - --split-by=timings \ - --verbose \ - --command="xargs python -m pytest \ - -v \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -k 'caching or cache'" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results auth_ui_unit_tests: docker: - image: cimg/python:3.11 @@ -664,376 +551,6 @@ jobs: # Store test results - store_test_results: path: test-results - litellm_security_tests: - docker: - - image: cimg/python:3.13 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - - image: cimg/postgres:14.0 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: circle_test - resource_class: xlarge - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/circle_test" - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - setup_remote_docker: - docker_layer_caching: true - - restore_cache: - keys: - - v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip uv - uv pip install --system -r requirements.txt - pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-mock==3.12.0" \ - "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" - - save_cache: - paths: - - ~/.local/lib - - ~/.local/bin - - ~/.cache/uv - key: v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m - - run: - name: Run Security Scans - command: | - chmod +x ci_cd/security_scans.sh - ./ci_cd/security_scans.sh - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - python -m pytest tests/proxy_security_tests -v -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 15m - # Store test results - - store_test_results: - path: test-results - # Split proxy unit tests into 3 jobs for faster execution and better debugging - # test_key_generate_prisma runs separately without parallel execution to avoid event loop issues with logging worker - litellm_proxy_unit_testing_key_generation: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: medium - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - run: - name: Install PostgreSQL - command: | - sudo apt-get update - sudo apt-get install -y postgresql-14 postgresql-contrib-14 - - restore_cache: - keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "pytest-timeout==2.2.0" - pip install "pytest-forked==1.6.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install "google-genai==1.22.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-postgresql==7.0.1" - pip install "fakeredis==2.28.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - run: - name: Run key generation tests (no parallel execution to avoid event loop issues) - command: | - pwd - ls - # Run without -n flag to avoid pytest-xdist event loop conflicts with logging worker - python -m pytest tests/proxy_unit_tests/test_key_generate_prisma.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-key-generation.xml --durations=10 --timeout=300 -vv --log-cli-level=INFO - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_proxy_unit_tests_key_generation_coverage.xml - mv .coverage litellm_proxy_unit_tests_key_generation_coverage - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_proxy_unit_tests_key_generation_coverage.xml - - litellm_proxy_unit_tests_key_generation_coverage - litellm_proxy_unit_testing_part1: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: xlarge - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - run: - name: Install PostgreSQL - command: | - sudo apt-get update - sudo apt-get install -y postgresql-14 postgresql-contrib-14 - - restore_cache: - keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "pytest-timeout==2.2.0" - pip install "pytest-forked==1.6.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install "google-genai==1.22.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-postgresql==7.0.1" - pip install "fakeredis==2.28.1" - pip install "pytest-xdist==3.6.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - run: - name: Run proxy unit tests (part 1 - auth checks) - command: | - pwd - ls - python -m pytest tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py --junitxml=test-results/junit-part1.xml --durations=10 -n 8 --timeout=300 -v - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_proxy_unit_testing_part2: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: xlarge - steps: - - checkout - - setup_google_dns - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - run: - name: Install PostgreSQL - command: | - sudo apt-get update - sudo apt-get install -y postgresql-14 postgresql-contrib-14 - - restore_cache: - keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "pytest-timeout==2.2.0" - pip install "pytest-forked==1.6.0" - pip install "mypy==1.18.2" - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install "google-genai==1.22.0" - pip install pyarrow - pip install "boto3==1.36.0" - pip install "aioboto3==13.4.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.59.7" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.100.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.22.0" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.10.2" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-postgresql==7.0.1" - pip install "fakeredis==2.28.1" - pip install "pytest-xdist==3.6.1" - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - run: - name: Run proxy unit tests (part 2 - remaining tests) - command: | - pwd - ls - python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --junitxml=test-results/junit-part2.xml --durations=10 -n 8 --timeout=300 -v - no_output_timeout: 15m - - store_test_results: - path: test-results litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword docker: - image: cimg/python:3.13.1 @@ -1115,7 +632,7 @@ jobs: for dir in "${IGNORE_DIRS[@]}"; do IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir" done - python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread + python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 no_output_timeout: 15m # Store test results @@ -1331,7 +848,7 @@ jobs: command: | pwd ls - python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5 no_output_timeout: 15m - run: name: Rename the coverage files @@ -1507,101 +1024,6 @@ jobs: no_output_timeout: 15m - store_test_results: path: test-results - litellm_mapped_tests_llms: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run LLM provider tests - command: | - python -m pytest tests/test_litellm/llms --junitxml=test-results/junit-llms.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_mapped_tests_core: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run core tests - command: | - python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --ignore=tests/test_litellm/experimental_mcp_client --junitxml=test-results/junit-core.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_mapped_tests_litellm_core_utils: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run litellm_core_utils tests - command: | - python -m pytest tests/test_litellm/litellm_core_utils --junitxml=test-results/junit-litellm-core-utils.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_mapped_tests_mcps: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: medium - steps: - - setup_litellm_test_deps - - run: - name: Run MCP client tests - command: | - python -m pytest tests/test_litellm/experimental_mcp_client --cov=litellm --cov-report=xml --junitxml=test-results/junit-mcps.xml --durations=10 -n 2 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_mcps_tests_coverage.xml - mv .coverage litellm_mcps_tests_coverage - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_mcps_tests_coverage.xml - - litellm_mcps_tests_coverage - litellm_mapped_tests_integrations: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run integrations tests - command: | - python -m pytest tests/test_litellm/integrations --junitxml=test-results/junit-integrations.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - no_output_timeout: 15m - - store_test_results: - path: test-results litellm_mapped_enterprise_tests: docker: - image: cimg/python:3.11 @@ -2137,6 +1559,25 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install aiohttp pip install apscheduler + - run: + name: Install dockerize + command: | + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=litellm_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m - attach_workspace: at: ~/project - run: @@ -2145,29 +1586,41 @@ jobs: zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: - name: Run Docker container + name: Seed database with real schema + command: | + docker run -d \ + -p 4001:4000 \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ + -e LITELLM_MASTER_KEY="sk-1234" \ + --name schema-seed \ + --add-host=host.docker.internal:host-gateway \ + -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --use_prisma_db_push + - run: + name: Wait for schema seed to complete + command: dockerize -wait http://localhost:4001 -timeout 5m + - run: + name: Stop schema seed container + command: docker stop schema-seed && docker rm schema-seed + - run: + name: Run Docker container with bad schema and disabled updates command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DISABLE_SCHEMA_UPDATE="True" \ + --name my-app \ + --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ - --name my-app \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: name: Wait for container to be ready command: dockerize -wait http://localhost:4000 -timeout 1m @@ -2575,9 +2028,6 @@ jobs: -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ - -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ - -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ - -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ @@ -2585,7 +2035,6 @@ jobs: -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e GCS_FLUSH_INTERVAL="1" \ --add-host host.docker.internal:host-gateway \ @@ -3061,6 +2510,20 @@ jobs: name: Build Docker image command: | docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip . + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: | + timeout 60s bash -c 'until docker exec postgres-db pg_isready -U postgres -d circle_test; do sleep 2; done' - run: name: Run Docker container # intentionally give bad redis credentials here @@ -3068,7 +2531,7 @@ jobs: command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ @@ -3076,18 +2539,15 @@ jobs: -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ - -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ - -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ - -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e GCS_FLUSH_INTERVAL="1" \ + --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \ my-app:latest \ @@ -3118,8 +2578,11 @@ jobs: - run: name: Stop and remove first container command: | - docker stop my-app - docker rm my-app + docker stop my-app || true + docker rm my-app || true + docker stop postgres-db || true + docker rm postgres-db || true + when: always proxy_pass_through_endpoint_tests: machine: image: ubuntu-2204:2023.10.1 @@ -3543,93 +3006,6 @@ jobs: - codecov/upload: file: ./coverage.xml - publish_to_pypi: - docker: - - image: cimg/python:3.8 - working_directory: ~/project - - environment: - TWINE_USERNAME: __token__ - - steps: - - checkout - - - run: - name: Copy model_prices_and_context_window File to model_prices_and_context_window_backup - command: | - cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json - - - run: - name: Checkout code - command: git checkout $CIRCLE_SHA1 - - # Check if setup.py is modified and publish to PyPI - - run: - name: PyPI publish - command: | - echo "Install TOML package." - python -m pip install toml - VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - PACKAGE_NAME=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['name'])") - if ! pip show -v $PACKAGE_NAME | grep -q "Version: ${VERSION}"; then - echo "pyproject.toml modified" - echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc - python -m pip install --upgrade pip - pip install build - pip install wheel - pip install --upgrade twine setuptools - rm -rf build dist - - echo "Building package" - python -m build - - echo "Twine upload to dist" - echo "Contents of dist directory:" - ls dist/ - twine upload --verbose dist/* - else - echo "Version ${VERSION} of package is already published on PyPI." - - # Check if corresponding Docker nightly image exists - NIGHTLY_TAG="v${VERSION}-nightly" - echo "Checking for Docker nightly image: litellm/litellm:${NIGHTLY_TAG}" - - # Check Docker Hub for the nightly image - if curl -s "https://hub.docker.com/v2/repositories/litellm/litellm/tags/${NIGHTLY_TAG}" | grep -q "name"; then - echo "Docker nightly image ${NIGHTLY_TAG} exists. This release was already completed successfully." - echo "Skipping PyPI publish and continuing to ensure Docker images are up to date." - circleci step halt - else - echo "ERROR: PyPI package ${VERSION} exists but Docker nightly image ${NIGHTLY_TAG} does not exist!" - echo "This indicates an incomplete release. Please investigate." - exit 1 - fi - fi - - run: - name: Trigger Github Action for new Docker Container + Trigger Load Testing - command: | - echo "Install TOML package." - python3 -m pip install toml - VERSION=$(python3 -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - echo "LiteLLM Version ${VERSION}" - - # Determine which branch to use for Docker build - if [[ "$CIRCLE_BRANCH" =~ ^litellm_release_day_.* ]]; then - BUILD_BRANCH="$CIRCLE_BRANCH" - echo "Using release branch: $BUILD_BRANCH" - else - BUILD_BRANCH="main" - echo "Using default branch: $BUILD_BRANCH" - fi - - curl -X POST \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - "https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \ - -d "{\"ref\":\"${BUILD_BRANCH}\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}" - echo "triggering load testing server for version ${VERSION} and commit ${CIRCLE_SHA1}" - curl -X POST "https://proxyloadtester-production.up.railway.app/start/load/test?version=${VERSION}&commit_hash=${CIRCLE_SHA1}&release_type=nightly" - publish_proxy_extras: docker: - image: cimg/python:3.8 @@ -3942,37 +3318,39 @@ jobs: - setup_google_dns - attach_workspace: at: ~/project + - run: + name: Install dockerize + command: | + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=litellm_schema_sync \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: - name: Install Neon CLI + name: Run schema sync via prisma db push command: | - npm i -g neonctl - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Sync schema on base e2e database - command: | - BASE_DATABASE_URL=$(neon connection-string \ - --project-id $NEON_PROJECT_ID \ - --api-key $NEON_API_KEY \ - --branch br-fancy-paper-ad1olsb3 \ - --database-name yuneng-trial-db \ - --role neondb_owner) docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$BASE_DATABASE_URL \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \ -e LITELLM_MASTER_KEY="sk-1234" \ --name schema-sync \ + --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ @@ -4089,34 +3467,14 @@ workflows: only: - main - /litellm_.*/ - - caching_unit_tests: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_proxy_unit_testing_key_generation: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_proxy_unit_testing_part1: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_proxy_unit_testing_part2: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_security_tests: - filters: - branches: - only: - main - /litellm_.*/ - litellm_assistants_api_testing: @@ -4170,7 +3528,6 @@ workflows: - main - /litellm_.*/ - prisma_schema_sync: - context: e2e_ui_tests requires: - build_docker_database_image filters: @@ -4178,32 +3535,32 @@ workflows: only: - main - /litellm_.*/ - - e2e_ui_testing: - name: e2e_ui_testing_chromium - browser: chromium - context: e2e_ui_tests - requires: - - ui_build - - build_docker_database_image - - prisma_schema_sync - filters: - branches: - only: - - main - - /litellm_.*/ - - e2e_ui_testing: - name: e2e_ui_testing_firefox - browser: firefox - context: e2e_ui_tests - requires: - - ui_build - - build_docker_database_image - - prisma_schema_sync - filters: - branches: - only: - - main - - /litellm_.*/ + # - e2e_ui_testing: # migrate to dynamic db - currently requires neon cli + # name: e2e_ui_testing_chromium + # browser: chromium + # context: e2e_ui_tests + # requires: + # - ui_build + # - build_docker_database_image + # - prisma_schema_sync + # filters: + # branches: + # only: + # - main + # - /litellm_.*/ + # - e2e_ui_testing: + # name: e2e_ui_testing_firefox + # browser: firefox + # context: e2e_ui_tests + # requires: + # - ui_build + # - build_docker_database_image + # - prisma_schema_sync + # filters: + # branches: + # only: + # - main + # - /litellm_.*/ - build_and_test: requires: - build_docker_database_image @@ -4352,34 +3709,14 @@ workflows: only: - main - /litellm_.*/ - - litellm_mapped_tests_llms: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_core: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_mcps: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_integrations: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_mapped_tests_litellm_core_utils: - filters: - branches: - only: - main - /litellm_.*/ - batches_testing: @@ -4429,11 +3766,6 @@ workflows: - search_testing - litellm_mapped_tests_proxy_part1 - litellm_mapped_tests_proxy_part2 - - litellm_mapped_tests_llms - - litellm_mapped_tests_core - - litellm_mapped_tests_mcps - - litellm_mapped_tests_integrations - - litellm_mapped_tests_litellm_core_utils - litellm_mapped_enterprise_tests - batches_testing - litellm_utils_testing @@ -4441,8 +3773,6 @@ workflows: - image_gen_testing - logging_testing - audio_testing - - caching_unit_tests - - litellm_proxy_unit_testing_key_generation - langfuse_logging_unit_tests - local_testing_part1 - local_testing_part2 @@ -4489,59 +3819,3 @@ workflows: only: - main - /litellm_release_day_.*/ - - publish_to_pypi: - requires: - - mypy_linting - - semgrep - - local_testing_part1 - - local_testing_part2 - - build_and_test - - e2e_openai_endpoints - - test_bad_database_url - - llm_translation_testing - - realtime_translation_testing - - mcp_testing - - agent_testing - - google_generate_content_endpoint_testing - - llm_responses_api_testing - - ocr_testing - - search_testing - - litellm_mapped_tests_proxy_part1 - - litellm_mapped_tests_proxy_part2 - - litellm_mapped_tests_llms - - litellm_mapped_tests_core - - litellm_mapped_tests_mcps - - litellm_mapped_tests_integrations - - litellm_mapped_tests_litellm_core_utils - - litellm_mapped_enterprise_tests - - batches_testing - - litellm_utils_testing - - pass_through_unit_testing - - image_gen_testing - - logging_testing - - audio_testing - - litellm_router_testing - - litellm_router_unit_testing - - caching_unit_tests - - langfuse_logging_unit_tests - - litellm_assistants_api_testing - - auth_ui_unit_tests - - ui_unit_tests - - db_migration_disable_update_check - - e2e_ui_testing_chromium - - e2e_ui_testing_firefox - - litellm_proxy_unit_testing_key_generation - - litellm_proxy_unit_testing_part1 - - litellm_proxy_unit_testing_part2 - - litellm_security_tests - - installing_litellm_on_python - - installing_litellm_on_python_3_13 - - proxy_logging_guardrails_model_info_tests - - proxy_spend_accuracy_tests - - proxy_multi_instance_tests - - proxy_store_model_in_db_tests - - proxy_build_from_pip_tests - - proxy_pass_through_endpoint_tests - - check_code_and_doc_quality - - publish_proxy_extras - - guardrails_testing diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index 8c1d85f96e..0000000000 --- a/.claude/settings.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(git show:*)", - "Bash(git worktree add:*)", - "Read(//Users/krrishdholakia/Documents/litellm/**)", - "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types/**)", - "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/**)", - "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/**)", - "Bash(python:*)", - "Bash(python -c \"\nimport sys; sys.path.insert\\(0, ''.''\\)\nfrom litellm.proxy.guardrails.guardrail_hooks.claude_code.guardrail import ClaudeCodeGuardrail, HOSTED_TOOL_PREFIXES\nprint\\(''HOSTED_TOOL_PREFIXES:'', HOSTED_TOOL_PREFIXES\\)\nprint\\(''ClaudeCodeGuardrail imported OK''\\)\n\")", - "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy/**)", - "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/**)", - "Bash(poetry run pytest:*)", - "Bash(git add:*)", - "Bash(git commit:*)", - "Bash(poetry run python:*)", - "Bash(poetry run pip:*)", - "Bash(git reset:*)", - "Bash(git cherry-pick:*)", - "Bash(git checkout:*)", - "Read(//Users/krrishdholakia/Documents/litellm/litellm/proxy/guardrails/guardrail_hooks/**)", - "Read(//Users/krrishdholakia/Documents/**)", - "Bash(git -C /Users/krrishdholakia/Documents/litellm-mcp-user-permissions worktree list)", - "Bash(ls:*)" - ], - "additionalDirectories": [ - "/Users/krrishdholakia/Documents/litellm-mcp-group-plan/plan", - "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/proxy/guardrails/guardrail_hooks/claude_code", - "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types", - "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails", - "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy", - "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/tests/test_litellm/proxy/auth" - ] - } -} diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 4744ab048c..cbf380bac0 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,7 +1,7 @@ blank_issues_enabled: true contact_links: - name: Schedule Demo - url: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions + url: https://enterprise.litellm.ai/demo about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM - name: Discord url: https://discord.com/invite/wuPM9dRgDw diff --git a/.github/actions/helm-oci-chart-releaser/action.yml b/.github/actions/helm-oci-chart-releaser/action.yml index 1823e26283..454c591d43 100644 --- a/.github/actions/helm-oci-chart-releaser/action.yml +++ b/.github/actions/helm-oci-chart-releaser/action.yml @@ -41,32 +41,54 @@ runs: using: composite steps: - name: Helm | Setup - uses: azure/setup-helm@v4 + uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 with: version: v3.20.0 - name: Helm | Login shell: bash - run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }} + env: + REGISTRY_PASSWORD: ${{ inputs.registry_password }} + REGISTRY_USERNAME: ${{ inputs.registry_username }} + REGISTRY: ${{ inputs.registry }} + run: echo "$REGISTRY_PASSWORD" | helm registry login -u "$REGISTRY_USERNAME" --password-stdin "$REGISTRY" - name: Helm | Dependency if: inputs.update_dependencies == 'true' shell: bash - run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} + env: + CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} + run: helm dependency update "$CHART_PATH" - name: Helm | Package shell: bash - run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }} + env: + CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} + TAG: ${{ inputs.tag }} + APP_VERSION: ${{ inputs.app_version }} + run: helm package "$CHART_PATH" --version "$TAG" --app-version "$APP_VERSION" - name: Helm | Push shell: bash - run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }} + env: + NAME: ${{ inputs.name }} + TAG: ${{ inputs.tag }} + REGISTRY: ${{ inputs.registry }} + REPOSITORY: ${{ inputs.repository }} + run: helm push "${NAME}-${TAG}.tgz" "oci://${REGISTRY}/${REPOSITORY}" - name: Helm | Logout shell: bash - run: helm registry logout ${{ inputs.registry }} + env: + REGISTRY: ${{ inputs.registry }} + run: helm registry logout "$REGISTRY" - name: Helm | Output id: output shell: bash - run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT + env: + REGISTRY: ${{ inputs.registry }} + REPOSITORY: ${{ inputs.repository }} + NAME: ${{ inputs.name }} + TAG: ${{ inputs.tag }} + run: echo "image=${REGISTRY}/${REPOSITORY}/${NAME}:${TAG}" >> $GITHUB_OUTPUT diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index 20807685e1..36d70c1d74 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -1,22 +1,21 @@ name: "LiteLLM CodeQL config" -# Use security-extended suite instead of security-and-quality to avoid -# result sets > 2 GiB on this codebase that cause fatal OOM failures. queries: - - uses: security-extended + - uses: security-and-quality -# These two queries are security queries included in security-extended that -# individually produce result sets > 2 GiB on this codebase, causing fatal -# OOM failures. Exclude them as a safety net until CI confirms they no longer -# OOM; drop these exclusions in a follow-up once verified. +# Known OOM queries on large Python codebases: +# CodeQL builds a full data flow graph in memory. These two queries trace +# sensitive data through every log call / regex pattern, causing combinatorial +# path explosion on codebases with extensive logging like LiteLLM (>2 GiB +# result sets). This is a known CodeQL scaling limitation, not a code issue. +# Re-test periodically as CodeQL improves or the codebase refactors logging. query-filters: - exclude: - id: py/clear-text-logging-sensitive-data # CWE-312 — > 2 GiB result set + id: py/clear-text-logging-sensitive-data # CWE-312 - exclude: - id: py/polynomial-redos # CWE-730 — > 2 GiB result set + id: py/polynomial-redos # CWE-730 paths-ignore: - tests - docs - "**/*.md" - - litellm/proxy/_experimental/out diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 58e7cfe10d..c49882a8d6 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -4,6 +4,9 @@ updates: directory: "/" schedule: interval: "daily" + cooldown: + default-days: 7 + semver-major-days: 14 groups: github-actions: patterns: diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml new file mode 100644 index 0000000000..f1ae30e67d --- /dev/null +++ b/.github/workflows/_test-unit-base.yml @@ -0,0 +1,96 @@ +name: _Unit Test Base (Reusable) + +on: + workflow_call: + inputs: + test-path: + description: "Pytest path(s) to run" + required: true + type: string + workers: + description: "Number of pytest-xdist workers" + required: false + type: number + default: 2 + reruns: + description: "Number of reruns for flaky tests" + required: false + type: number + default: 2 + timeout-minutes: + description: "Job timeout in minutes" + required: false + type: number + default: 20 + max-failures: + description: "Stop after this many failures" + required: false + type: number + default: 10 + +permissions: + contents: read + +jobs: + run: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.timeout-minutes }} + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run tests + env: + TEST_PATH: ${{ inputs.test-path }} + MAX_FAILURES: ${{ inputs.max-failures }} + WORKERS: ${{ inputs.workers }} + RERUNS: ${{ inputs.reruns }} + run: | + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + -n "${WORKERS}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml new file mode 100644 index 0000000000..d53a9e8822 --- /dev/null +++ b/.github/workflows/_test-unit-services-base.yml @@ -0,0 +1,164 @@ +name: _Unit Test Services Base (Reusable) + +on: + workflow_call: + inputs: + test-path: + description: "Pytest path(s) to run" + required: true + type: string + workers: + description: "Number of pytest-xdist workers (0 = no parallelism)" + required: false + type: number + default: 2 + reruns: + description: "Number of reruns for flaky tests" + required: false + type: number + default: 2 + timeout-minutes: + description: "Job timeout in minutes" + required: false + type: number + default: 20 + max-failures: + description: "Stop after this many failures" + required: false + type: number + default: 10 + enable-redis: + description: "Pass Redis Cloud credentials to tests via REDIS_HOST/PORT/PASSWORD env vars" + required: false + type: boolean + default: false + enable-postgres: + description: "Start a local Postgres service container and run Prisma migrations" + required: false + type: boolean + default: false + secrets: + REDIS_HOST: + required: false + REDIS_PORT: + required: false + REDIS_PASSWORD: + required: false + DATABASE_URL: + required: false + POSTGRES_USER: + required: false + POSTGRES_PASSWORD: + required: false + +permissions: + contents: read + +jobs: + run: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.timeout-minutes }} + # Environment is derived from the enable-* flags, not caller-controllable. + # This prevents callers from passing arbitrary environment names to bypass secret scoping. + # Note: Postgres service container always starts (GHA limitation), so any Redis job + # also needs Postgres secrets → uses integration-redis-postgres, not integration-redis. + environment: >- + ${{ + inputs.enable-redis && 'integration-redis-postgres' || + inputs.enable-postgres && 'integration-postgres' || + '' + }} + + services: + postgres: + image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14 + env: + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + POSTGRES_DB: litellm_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-services-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry-services- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run Prisma migrations + if: ${{ inputs.enable-postgres }} + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + run: | + poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + + - name: Run tests + env: + TEST_PATH: ${{ inputs.test-path }} + MAX_FAILURES: ${{ inputs.max-failures }} + WORKERS: ${{ inputs.workers }} + RERUNS: ${{ inputs.reruns }} + DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }} + REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }} + REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }} + REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }} + run: | + if [ "${WORKERS}" = "0" ]; then + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --durations=20 + else + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + -n "${WORKERS}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 + fi diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml index 98b9d868e6..60e8993621 100644 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -2,18 +2,24 @@ name: Updates model_prices_and_context_window.json and Create Pull Request on: schedule: - - cron: "0 0 * * 0" # Run every Sundays at midnight + - cron: "0 0 * * 0" # Run every Sundays at midnight #- cron: "0 0 * * *" # Run daily at midnight +permissions: + contents: write + pull-requests: write + jobs: auto_update_price_and_context_window: if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Install Dependencies run: | - pip install aiohttp + pip install 'aiohttp==3.13.3' - name: Update JSON Data run: | python ".github/workflows/auto_update_price_and_context_window_file.py" @@ -26,4 +32,4 @@ jobs: --head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \ --base main env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} \ No newline at end of file + GH_TOKEN: ${{ secrets.GH_TOKEN }} diff --git a/.github/workflows/check-schema-sync.yml b/.github/workflows/check-schema-sync.yml new file mode 100644 index 0000000000..0e5e2804e6 --- /dev/null +++ b/.github/workflows/check-schema-sync.yml @@ -0,0 +1,58 @@ +name: Check Schema Sync + +on: + pull_request: + paths: + - 'schema.prisma' + - 'litellm/proxy/schema.prisma' + - 'litellm-proxy-extras/litellm_proxy_extras/schema.prisma' + +permissions: + contents: read + +jobs: + check-sync: + name: Verify schema.prisma copies match root + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout PR + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Reject symlinked schema files + run: | + for f in schema.prisma litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; do + if [ -L "$f" ]; then + echo "::error file=$f::$f is a symlink, which is not allowed" + exit 1 + fi + done + + - name: Check all schemas match root + run: | + EXIT=0 + + diff schema.prisma litellm/proxy/schema.prisma || { + echo "::error file=litellm/proxy/schema.prisma::litellm/proxy/schema.prisma differs from root schema.prisma" + EXIT=1 + } + + diff schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma || { + echo "::error file=litellm-proxy-extras/litellm_proxy_extras/schema.prisma::litellm-proxy-extras/litellm_proxy_extras/schema.prisma differs from root schema.prisma" + EXIT=1 + } + + if [ "$EXIT" -ne 0 ]; then + echo "" + echo "Schema files are out of sync." + echo "The root schema.prisma is the source of truth." + echo "" + echo "To fix, run from the repo root:" + echo " cp schema.prisma litellm/proxy/schema.prisma" + echo " cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma" + exit 1 + fi + + echo "All schema copies are in sync with root." diff --git a/.github/workflows/check_duplicate_issues.yml b/.github/workflows/check_duplicate_issues.yml index 6d11ce573e..289d78880a 100644 --- a/.github/workflows/check_duplicate_issues.yml +++ b/.github/workflows/check_duplicate_issues.yml @@ -12,7 +12,7 @@ jobs: contents: read steps: - name: Check for potential duplicates - uses: wow-actions/potential-duplicates@v1 + uses: wow-actions/potential-duplicates@4d4ea0352e0383859279938e255179dd1dbb67b5 # v1.1.0 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} label: potential-duplicate @@ -30,13 +30,14 @@ jobs: - name: Checkout close script if: github.event.action == 'opened' - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: sparse-checkout: .github/scripts + persist-credentials: false - name: Set up Python if: github.event.action == 'opened' - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0b7cce2e4b..e86fca17c7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -6,8 +6,8 @@ on: pull_request: branches: [main] schedule: - # Run weekly on Sundays at 04:00 UTC - - cron: "0 4 * * 0" + # Run daily at 04:00 UTC + - cron: "0 4 * * *" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -15,6 +15,7 @@ concurrency: jobs: analyze: + if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm' name: Analyze (${{ matrix.language }}) runs-on: ubuntu-latest timeout-minutes: 30 @@ -37,16 +38,18 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} config-file: ./.github/codeql/codeql-config.yml - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 385b95fdaf..52d64addea 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -25,10 +25,12 @@ jobs: timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -38,7 +40,7 @@ jobs: pip install pytest pytest-codspeed==4.3.0 - name: Run benchmarks - uses: CodSpeedHQ/action@v4 + uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1 with: mode: simulation run: pytest tests/benchmarks/ --codspeed diff --git a/.github/workflows/create_daily_staging_branch.yml b/.github/workflows/create_daily_staging_branch.yml index 08aebd7d04..424d8de0a4 100644 --- a/.github/workflows/create_daily_staging_branch.yml +++ b/.github/workflows/create_daily_staging_branch.yml @@ -2,18 +2,22 @@ name: Create Daily Staging Branch on: schedule: - - cron: '0 0,12 * * *' # Runs every 12 hours at midnight and noon UTC - workflow_dispatch: # Allow manual trigger + - cron: "0 0,12 * * *" # Runs every 12 hours at midnight and noon UTC + workflow_dispatch: # Allow manual trigger jobs: create-staging-branch: + if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: fetch-depth: 0 + persist-credentials: false - name: Create daily staging branch env: @@ -43,13 +47,17 @@ jobs: fi create-internal-dev-branch: + if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: fetch-depth: 0 + persist-credentials: false - name: Create internal dev branch env: diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml deleted file mode 100644 index 344b0ec48e..0000000000 --- a/.github/workflows/ghcr_deploy.yml +++ /dev/null @@ -1,444 +0,0 @@ -# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM -name: Build, Publish LiteLLM Docker Image. New Release -on: - workflow_dispatch: - inputs: - tag: - description: "The tag version you want to build" - required: true - release_type: - description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'" - type: string - default: "latest" - commit_hash: - description: "Commit hash" - required: true - -# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - CHART_NAME: litellm-helm - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - # print commit hash, tag, and release type - print: - runs-on: ubuntu-latest - steps: - - run: | - echo "Commit hash: ${{ github.event.inputs.commit_hash }}" - echo "Tag: ${{ github.event.inputs.tag }}" - echo "Release type: ${{ github.event.inputs.release_type }}" - docker-hub-deploy: - if: github.repository == 'BerriAI/litellm' - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: . - push: true - tags: litellm/litellm:${{ github.event.inputs.tag || 'latest' }} - - - name: Build and push litellm-database image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: ./docker/Dockerfile.database - tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }} - - - name: Build and push litellm-spend-logs image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: ./litellm-js/spend-logs/Dockerfile - tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }} - - - name: Build and push litellm-non_root image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: ./docker/Dockerfile.non_root - tags: litellm/litellm-non_root:${{ github.event.inputs.tag || 'latest' }} - build-and-push-image: - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Build and push Docker image - uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8 - with: - context: . - push: true - tags: | - ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }}, - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - labels: ${{ steps.meta.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-ee: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for EE Dockerfile - id: meta-ee - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push EE Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Dockerfile - push: true - tags: | - ${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-ee:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-stable', env.REGISTRY) || '' }} - labels: ${{ steps.meta-ee.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-database: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for database Dockerfile - id: meta-database - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-database - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push Database Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: ./docker/Dockerfile.database - push: true - tags: | - ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-stable', env.REGISTRY) || '' }} - labels: ${{ steps.meta-database.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-non_root: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for non_root Dockerfile - id: meta-non_root - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-non_root - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push non_root Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: ./docker/Dockerfile.non_root - push: true - tags: | - ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-stable', env.REGISTRY) || '' }} - labels: ${{ steps.meta-non_root.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-spend-logs: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for spend-logs Dockerfile - id: meta-spend-logs - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-spend_logs - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push Database Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: ./litellm-js/spend-logs/Dockerfile - push: true - tags: | - ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, - ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }} - ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }}, - ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - run-observatory-tests: - if: github.event.inputs.release_type == 'rc' || github.event.inputs.release_type == 'stable' - needs: [docker-hub-deploy] - uses: ./.github/workflows/run_observatory_tests.yml - with: - tag: ${{ github.event.inputs.tag }} - commit_hash: ${{ github.event.inputs.commit_hash }} - secrets: inherit - - build-and-push-helm-chart: - if: github.event.inputs.release_type != 'dev' - needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database] - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: lowercase github.repository_owner - run: | - echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV} - - # Sync Helm chart version with LiteLLM release version (1-1 versioning) - # This allows users to easily map Helm chart versions to LiteLLM versions - # See: https://codefresh.io/docs/docs/ci-cd-guides/helm-best-practices/ - - name: Calculate chart and app versions - id: chart_version - shell: bash - run: | - INPUT_TAG="${{ github.event.inputs.tag }}" - RELEASE_TYPE="${{ github.event.inputs.release_type }}" - - # Chart version = LiteLLM version without 'v' prefix (Helm semver convention) - # v1.81.0 -> 1.81.0, v1.81.0.rc.1 -> 1.81.0.rc.1 - CHART_VERSION="${INPUT_TAG#v}" - - # Add suffix for 'latest' releases (rc already has suffix in tag) - if [ "$RELEASE_TYPE" = "latest" ]; then - CHART_VERSION="${CHART_VERSION}-latest" - fi - - # App version = Docker tag (keeps 'v' prefix to match Docker image tags) - APP_VERSION="${INPUT_TAG}" - - echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT - echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT - - - uses: ./.github/actions/helm-oci-chart-releaser - with: - name: ${{ env.CHART_NAME }} - repository: ${{ env.REPO_OWNER }} - tag: ${{ steps.chart_version.outputs.version }} - app_version: ${{ steps.chart_version.outputs.app_version }} - path: deploy/charts/${{ env.CHART_NAME }} - registry: ${{ env.REGISTRY }} - registry_username: ${{ github.actor }} - registry_password: ${{ secrets.GITHUB_TOKEN }} - update_dependencies: true - - release: - name: "New LiteLLM Release" - needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database] - permissions: - contents: write - runs-on: "ubuntu-latest" - - steps: - - name: Display version - run: echo "Current version is ${{ github.event.inputs.tag }}" - - name: "Set Release Tag" - run: echo "RELEASE_TAG=${{ github.event.inputs.tag }}" >> $GITHUB_ENV - - name: Display release tag - run: echo "RELEASE_TAG is $RELEASE_TAG" - - name: "Create release" - uses: "actions/github-script@v6" - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - script: | - const commitHash = "${{ github.event.inputs.commit_hash}}"; - console.log("Commit Hash:", commitHash); // Add this line for debugging - try { - const response = await github.rest.repos.createRelease({ - draft: false, - generate_release_notes: true, - target_commitish: commitHash, - name: process.env.RELEASE_TAG, - owner: context.repo.owner, - prerelease: false, - repo: context.repo.repo, - tag_name: process.env.RELEASE_TAG, - }); - - core.exportVariable('RELEASE_ID', response.data.id); - core.exportVariable('RELEASE_UPLOAD_URL', response.data.upload_url); - } catch (error) { - core.setFailed(error.message); - } - - name: Fetch Release Notes - id: release-notes - uses: actions/github-script@v6 - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - script: | - try { - const response = await github.rest.repos.getRelease({ - owner: context.repo.owner, - repo: context.repo.repo, - release_id: process.env.RELEASE_ID, - }); - const formattedBody = JSON.stringify(response.data.body).slice(1, -1); - return formattedBody; - } catch (error) { - core.setFailed(error.message); - } - env: - RELEASE_ID: ${{ env.RELEASE_ID }} - - name: Github Releases To Discord - env: - WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }} - REALEASE_TAG: ${{ env.RELEASE_TAG }} - RELEASE_NOTES: ${{ steps.release-notes.outputs.result }} - run: | - curl -H "Content-Type: application/json" -X POST -d '{ - "content": "New LiteLLM release '"${RELEASE_TAG}"'", - "username": "Release Changelog", - "avatar_url": "https://cdn.discordapp.com/avatars/487431320314576937/bd64361e4ba6313d561d54e78c9e7171.png", - "embeds": [ - { - "title": "Changelog for LiteLLM '"${RELEASE_TAG}"'", - "description": "'"${RELEASE_NOTES}"'", - "color": 2105893 - } - ] - }' $WEBHOOK_URL - diff --git a/.github/workflows/ghcr_helm_deploy.yml b/.github/workflows/ghcr_helm_deploy.yml deleted file mode 100644 index 21b2eaafe1..0000000000 --- a/.github/workflows/ghcr_helm_deploy.yml +++ /dev/null @@ -1,67 +0,0 @@ -# Standalone workflow to publish LiteLLM Helm Chart -# Note: The main ghcr_deploy.yml workflow also publishes the Helm chart as part of a full release -name: Build, Publish LiteLLM Helm Chart. New Release -on: - workflow_dispatch: - inputs: - tag: - description: "LiteLLM version tag (e.g., v1.81.0)" - required: true - -# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - REPO_OWNER: ${{github.repository_owner}} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-helm-chart: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: lowercase github.repository_owner - run: | - echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV} - - # Sync Helm chart version with LiteLLM release version (1-1 versioning) - - name: Calculate chart and app versions - id: chart_version - shell: bash - run: | - INPUT_TAG="${{ github.event.inputs.tag }}" - - # Chart version = LiteLLM version without 'v' prefix - # v1.81.0 -> 1.81.0 - CHART_VERSION="${INPUT_TAG#v}" - - # App version = Docker tag (keeps 'v' prefix) - APP_VERSION="${INPUT_TAG}" - - echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT - echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT - - - name: Lint helm chart - run: helm lint deploy/charts/litellm-helm - - - uses: ./.github/actions/helm-oci-chart-releaser - with: - name: litellm-helm - repository: ${{ env.REPO_OWNER }} - tag: ${{ steps.chart_version.outputs.version }} - app_version: ${{ steps.chart_version.outputs.app_version }} - path: deploy/charts/litellm-helm - registry: ${{ env.REGISTRY }} - registry_username: ${{ github.actor }} - registry_password: ${{ secrets.GITHUB_TOKEN }} - update_dependencies: true - diff --git a/.github/workflows/helm_unit_test.yml b/.github/workflows/helm_unit_test.yml index c4b83af70a..06836b1d1c 100644 --- a/.github/workflows/helm_unit_test.yml +++ b/.github/workflows/helm_unit_test.yml @@ -6,22 +6,36 @@ on: branches: - main +permissions: + contents: read + jobs: unit-test: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Helm 3.11.1 - uses: azure/setup-helm@v1 + uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 with: - version: '3.11.1' + version: "3.11.1" - name: Install Helm Unit Test Plugin run: | helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 + - name: Verify Helm Unit Test Plugin integrity + run: | + EXPECTED_SHA="e251ba198448629678ff2168e1a469249d998155" + PLUGIN_DIR="$(helm env HELM_PLUGINS)/helm-unittest" + ACTUAL_SHA="$(git -C "$PLUGIN_DIR" rev-parse HEAD)" + if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then + echo "::error::Helm unittest plugin checksum mismatch! Expected $EXPECTED_SHA but got $ACTUAL_SHA" + exit 1 + fi + echo "Helm unittest plugin integrity verified: $ACTUAL_SHA" - name: Run unit tests - run: - helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm \ No newline at end of file + run: helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm diff --git a/.github/workflows/interpret_load_test.py b/.github/workflows/interpret_load_test.py deleted file mode 100644 index 348ff300ff..0000000000 --- a/.github/workflows/interpret_load_test.py +++ /dev/null @@ -1,139 +0,0 @@ -import csv -import os -from github import Github - - -def interpret_results(csv_file): - with open(csv_file, newline="") as csvfile: - csvreader = csv.DictReader(csvfile) - rows = list(csvreader) - """ - in this csv reader - - Create 1 new column "Status" - - if a row has a median response time < 300 and an average response time < 300, Status = "Passed ✅" - - if a row has a median response time >= 300 or an average response time >= 300, Status = "Failed ❌" - - Order the table in this order Name, Status, Median Response Time, Average Response Time, Requests/s,Failures/s, Min Response Time, Max Response Time, all other columns - """ - - # Add a new column "Status" - for row in rows: - median_response_time = float( - row["Median Response Time"].strip().rstrip("ms") - ) - average_response_time = float( - row["Average Response Time"].strip().rstrip("s") - ) - - request_count = int(row["Request Count"]) - failure_count = int(row["Failure Count"]) - - failure_percent = round((failure_count / request_count) * 100, 2) - - # Determine status based on conditions - if ( - median_response_time < 300 - and average_response_time < 300 - and failure_percent < 5 - ): - row["Status"] = "Passed ✅" - else: - row["Status"] = "Failed ❌" - - # Construct Markdown table header - markdown_table = "| Name | Status | Median Response Time (ms) | Average Response Time (ms) | Requests/s | Failures/s | Request Count | Failure Count | Min Response Time (ms) | Max Response Time (ms) |" - markdown_table += ( - "\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |" - ) - - # Construct Markdown table rows - for row in rows: - markdown_table += f"\n| {row['Name']} | {row['Status']} | {row['Median Response Time']} | {row['Average Response Time']} | {row['Requests/s']} | {row['Failures/s']} | {row['Request Count']} | {row['Failure Count']} | {row['Min Response Time']} | {row['Max Response Time']} |" - print("markdown table: ", markdown_table) - return markdown_table - - -def _get_docker_run_command_stable_release(release_version): - return f""" -\n\n -## Docker Run LiteLLM Proxy - -``` -docker run \\ --e STORE_MODEL_IN_DB=True \\ --p 4000:4000 \\ -ghcr.io/berriai/litellm:litellm_stable_release_branch-{release_version} -``` - """ - - -def _get_docker_run_command(release_version): - return f""" -\n\n -## Docker Run LiteLLM Proxy - -``` -docker run \\ --e STORE_MODEL_IN_DB=True \\ --p 4000:4000 \\ -ghcr.io/berriai/litellm:main-{release_version} -``` - """ - - -def get_docker_run_command(release_version): - if "stable" in release_version: - return _get_docker_run_command_stable_release(release_version) - else: - return _get_docker_run_command(release_version) - - -if __name__ == "__main__": - return - csv_file = "load_test_stats.csv" # Change this to the path of your CSV file - markdown_table = interpret_results(csv_file) - - # Update release body with interpreted results - github_token = os.getenv("GITHUB_TOKEN") - g = Github(github_token) - repo = g.get_repo( - "BerriAI/litellm" - ) # Replace with your repository's username and name - latest_release = repo.get_latest_release() - print("got latest release: ", latest_release) - print(latest_release.title) - print(latest_release.tag_name) - - release_version = latest_release.title - - print("latest release body: ", latest_release.body) - print("markdown table: ", markdown_table) - - # check if "Load Test LiteLLM Proxy Results" exists - existing_release_body = latest_release.body - if "Load Test LiteLLM Proxy Results" in latest_release.body: - # find the "Load Test LiteLLM Proxy Results" section and delete it - start_index = latest_release.body.find("Load Test LiteLLM Proxy Results") - existing_release_body = latest_release.body[:start_index] - - docker_run_command = get_docker_run_command(release_version) - print("docker run command: ", docker_run_command) - - new_release_body = ( - existing_release_body - + docker_run_command - + "\n\n" - + "### Don't want to maintain your internal proxy? get in touch 🎉" - + "\nHosted Proxy Alpha: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions" - + "\n\n" - + "## Load Test LiteLLM Proxy Results" - + "\n\n" - + markdown_table - ) - print("new release body: ", new_release_body) - try: - latest_release.update_release( - name=latest_release.tag_name, - message=new_release_body, - ) - except Exception as e: - print(e) diff --git a/.github/workflows/issue-keyword-labeler.yml b/.github/workflows/issue-keyword-labeler.yml index 936f90f747..7e2693209b 100644 --- a/.github/workflows/issue-keyword-labeler.yml +++ b/.github/workflows/issue-keyword-labeler.yml @@ -2,8 +2,8 @@ name: Issue Keyword Labeler on: issues: - types: - - opened + types: + - opened jobs: scan-and-label: @@ -13,7 +13,9 @@ jobs: contents: read steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Scan for provider keywords id: scan @@ -24,7 +26,7 @@ jobs: - name: Ensure label exists if: steps.scan.outputs.found == 'true' - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -51,7 +53,7 @@ jobs: - name: Add label to the issue if: steps.scan.outputs.found == 'true' - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -61,4 +63,3 @@ jobs: issue_number: context.issue.number, labels: ['llm translation'] }); - diff --git a/.github/workflows/label-component.yml b/.github/workflows/label-component.yml index fd079fce6c..e0c2fa94d8 100644 --- a/.github/workflows/label-component.yml +++ b/.github/workflows/label-component.yml @@ -12,7 +12,7 @@ jobs: issues: write steps: - name: Add component labels - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/llm-translation-testing.yml b/.github/workflows/llm-translation-testing.yml index 7fda37a66d..922013c4b5 100644 --- a/.github/workflows/llm-translation-testing.yml +++ b/.github/workflows/llm-translation-testing.yml @@ -4,38 +4,41 @@ on: workflow_dispatch: inputs: release_candidate_tag: - description: 'Release candidate tag/version' + description: "Release candidate tag/version" required: true type: string push: tags: - - 'v*-rc*' # Triggers on release candidate tags like v1.0.0-rc1 - + - "v*-rc*" # Triggers on release candidate tags like v1.0.0-rc1 + +permissions: + contents: read + jobs: run-llm-translation-tests: runs-on: ubuntu-latest timeout-minutes: 90 - + steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: + persist-credentials: false ref: ${{ github.event.inputs.release_candidate_tag || github.ref }} - + - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: '3.11' - + python-version: "3.11" + - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: latest - virtualenvs-create: true - virtualenvs-in-project: true - - - name: Cache Poetry dependencies - uses: actions/cache@v3 + run: | + pip install 'poetry==2.3.2' + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + + - name: Restore Poetry dependencies cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 with: path: | ~/.cache/pypoetry @@ -43,15 +46,15 @@ jobs: key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} restore-keys: | ${{ runner.os }}-poetry- - + - name: Install dependencies run: | poetry install --with dev - poetry run pip install pytest-xdist pytest-timeout - + poetry run pip install 'pytest-xdist==3.8.0' 'pytest-timeout==2.4.0' + - name: Create test results directory run: mkdir -p test-results - + - name: Run LLM Translation Tests env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -61,13 +64,14 @@ jobs: AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }} - # Add other API keys as needed + RC_TAG: ${{ github.event.inputs.release_candidate_tag || github.ref_name }} + COMMIT_SHA: ${{ github.sha }} run: | python .github/workflows/run_llm_translation_tests.py \ - --tag "${{ github.event.inputs.release_candidate_tag || github.ref_name }}" \ - --commit "${{ github.sha }}" \ + --tag "$RC_TAG" \ + --commit "$COMMIT_SHA" \ || true # Continue even if tests fail - + - name: Display test summary if: always() run: | @@ -79,9 +83,9 @@ jobs: else echo "Warning: Test report was not generated" fi - + - name: Upload test artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() with: name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }} diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml deleted file mode 100644 index cdaffa328c..0000000000 --- a/.github/workflows/load_test.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Test Locust Load Test - -on: - workflow_run: - workflows: ["Build, Publish LiteLLM Docker Image. New Release"] - types: - - completed - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install PyGithub - - name: re-deploy proxy - run: | - echo "Current working directory: $PWD" - ls - python ".github/workflows/redeploy_proxy.py" - env: - LOAD_TEST_REDEPLOY_URL1: ${{ secrets.LOAD_TEST_REDEPLOY_URL1 }} - LOAD_TEST_REDEPLOY_URL2: ${{ secrets.LOAD_TEST_REDEPLOY_URL2 }} - working-directory: ${{ github.workspace }} - - name: Run Load Test - id: locust_run - uses: BerriAI/locust-github-action@master - with: - LOCUSTFILE: ".github/workflows/locustfile.py" - URL: "https://post-release-load-test-proxy.onrender.com/" - USERS: "20" - RATE: "20" - RUNTIME: "300s" - - name: Process Load Test Stats - run: | - echo "Current working directory: $PWD" - ls - python ".github/workflows/interpret_load_test.py" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: ${{ github.workspace }} - - name: Upload CSV as Asset to Latest Release - uses: xresloader/upload-to-github-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - file: "load_test_stats.csv;load_test.html" - update_latest_release: true - tag_name: "load-test" - overwrite: true \ No newline at end of file diff --git a/.github/workflows/locustfile.py b/.github/workflows/locustfile.py deleted file mode 100644 index 36dbeee9c4..0000000000 --- a/.github/workflows/locustfile.py +++ /dev/null @@ -1,28 +0,0 @@ -from locust import HttpUser, task, between - - -class MyUser(HttpUser): - wait_time = between(1, 5) - - @task - def chat_completion(self): - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer sk-8N1tLOOyH8TIxwOLahhIVg", - # Include any additional headers you may need for authentication, etc. - } - - # Customize the payload with "model" and "messages" keys - payload = { - "model": "fake-openai-endpoint", - "messages": [ - {"role": "system", "content": "You are a chat bot."}, - {"role": "user", "content": "Hello, how are you?"}, - ], - # Add more data as necessary - } - - # Make a POST request to the "chat/completions" endpoint - response = self.client.post("chat/completions", json=payload, headers=headers) - - # Print or log the response if needed diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 23e4a06da9..0000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Publish Dev Release to PyPI - -on: - workflow_dispatch: - -jobs: - publish-dev-release: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 # Adjust the Python version as needed - - - name: Install dependencies - run: pip install toml twine - - - name: Read version from pyproject.toml - id: read-version - run: | - version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])') - printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV - - - name: Check if version exists on PyPI - id: check-version - run: | - set -e - if twine check --repository-url https://pypi.org/simple/ "litellm==$LITELLM_VERSION" >/dev/null 2>&1; then - echo "Version $LITELLM_VERSION already exists on PyPI. Skipping publish." - diff --git a/.github/workflows/publish-migrations.yml b/.github/workflows/publish-migrations.yml deleted file mode 100644 index a5187cb2f5..0000000000 --- a/.github/workflows/publish-migrations.yml +++ /dev/null @@ -1,207 +0,0 @@ -name: Publish Prisma Migrations - -permissions: - contents: write - pull-requests: write - -on: - push: - paths: - - 'schema.prisma' # Check root schema.prisma - branches: - - main - -jobs: - publish-migrations: - if: github.repository == 'BerriAI/litellm' - runs-on: ubuntu-latest - services: - postgres: - image: postgres:14 - env: - POSTGRES_DB: temp_db - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - # Add shadow database service - postgres_shadow: - image: postgres:14 - env: - POSTGRES_DB: shadow_db - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: - - 5433:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - - name: Install Dependencies - run: | - pip install prisma - pip install python-dotenv - - - name: Generate Initial Migration if None Exists - env: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db" - run: | - mkdir -p deploy/migrations - echo 'provider = "postgresql"' > deploy/migrations/migration_lock.toml - - if [ -z "$(ls -A deploy/migrations/2* 2>/dev/null)" ]; then - echo "No existing migrations found, creating baseline..." - VERSION=$(date +%Y%m%d%H%M%S) - mkdir -p deploy/migrations/${VERSION}_initial - - echo "Generating initial migration..." - # Save raw output for debugging - prisma migrate diff \ - --from-empty \ - --to-schema-datamodel schema.prisma \ - --shadow-database-url "${SHADOW_DATABASE_URL}" \ - --script > deploy/migrations/${VERSION}_initial/raw_migration.sql - - echo "Raw migration file content:" - cat deploy/migrations/${VERSION}_initial/raw_migration.sql - - echo "Cleaning migration file..." - # Clean the file - sed '/^Installing/d' deploy/migrations/${VERSION}_initial/raw_migration.sql > deploy/migrations/${VERSION}_initial/migration.sql - - # Verify the migration file - if [ ! -s deploy/migrations/${VERSION}_initial/migration.sql ]; then - echo "ERROR: Migration file is empty after cleaning" - echo "Original content was:" - cat deploy/migrations/${VERSION}_initial/raw_migration.sql - exit 1 - fi - - echo "Final migration file content:" - cat deploy/migrations/${VERSION}_initial/migration.sql - - # Verify it starts with SQL - if ! head -n 1 deploy/migrations/${VERSION}_initial/migration.sql | grep -q "^--\|^CREATE\|^ALTER"; then - echo "ERROR: Migration file does not start with SQL command or comment" - echo "First line is:" - head -n 1 deploy/migrations/${VERSION}_initial/migration.sql - echo "Full content is:" - cat deploy/migrations/${VERSION}_initial/migration.sql - exit 1 - fi - - echo "Initial migration generated at $(date -u)" > deploy/migrations/${VERSION}_initial/README.md - fi - - - name: Compare and Generate Migration - if: success() - env: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db" - run: | - # Create temporary migration workspace - mkdir -p temp_migrations - - # Copy existing migrations (will not fail if directory is empty) - cp -r deploy/migrations/* temp_migrations/ 2>/dev/null || true - - VERSION=$(date +%Y%m%d%H%M%S) - - # Generate diff against existing migrations or empty state - prisma migrate diff \ - --from-migrations temp_migrations \ - --to-schema-datamodel schema.prisma \ - --shadow-database-url "${SHADOW_DATABASE_URL}" \ - --script > temp_migrations/migration_${VERSION}.sql - - # Check if there are actual changes - if [ -s temp_migrations/migration_${VERSION}.sql ]; then - echo "Changes detected, creating new migration" - mkdir -p deploy/migrations/${VERSION}_schema_update - mv temp_migrations/migration_${VERSION}.sql deploy/migrations/${VERSION}_schema_update/migration.sql - echo "Migration generated at $(date -u)" > deploy/migrations/${VERSION}_schema_update/README.md - else - echo "No schema changes detected" - exit 0 - fi - - - name: Verify Migration - if: success() - env: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db" - SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db" - run: | - # Create test database - psql "${SHADOW_DATABASE_URL}" -c 'CREATE DATABASE migration_test;' - - # Apply all migrations in order to verify - for migration in deploy/migrations/*/migration.sql; do - echo "Applying migration: $migration" - psql "${SHADOW_DATABASE_URL}" -f $migration - done - - # Add this step before create-pull-request to debug permissions - - name: Check Token Permissions - run: | - echo "Checking token permissions..." - curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/BerriAI/litellm/collaborators - - echo "\nChecking if token can create PRs..." - curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/BerriAI/litellm - - # Add this debug step before git push - - name: Debug Changed Files - run: | - echo "Files staged for commit:" - git diff --name-status --staged - - echo "\nAll changed files:" - git status - - - name: Create Pull Request - if: success() - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.GITHUB_TOKEN }} - commit-message: "chore: update prisma migrations" - title: "Update Prisma Migrations" - body: | - Auto-generated migration based on schema.prisma changes. - - Generated files: - - deploy/migrations/${VERSION}_schema_update/migration.sql - - deploy/migrations/${VERSION}_schema_update/README.md - branch: feat/prisma-migration-${{ env.VERSION }} - base: main - delete-branch: true - - - name: Generate and Save Migrations - run: | - # Only add migration files - git add deploy/migrations/ - git status # Debug what's being committed - git commit -m "chore: update prisma migrations" diff --git a/.github/workflows/publish_enterprise.yml b/.github/workflows/publish_enterprise.yml deleted file mode 100644 index 459a233cb7..0000000000 --- a/.github/workflows/publish_enterprise.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Publish litellm-enterprise to PyPI - -on: - workflow_dispatch: - inputs: - bump: - description: "Version bump type" - required: true - default: "patch" - type: choice - options: - - patch - - minor - - major - -jobs: - publish: - runs-on: ubuntu-latest - if: github.repository == 'BerriAI/litellm' - permissions: - contents: write - pull-requests: write - defaults: - run: - working-directory: enterprise - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Poetry - run: pip install poetry - - - name: Bump version - id: bump - run: | - OLD=$(poetry version -s) - poetry version ${{ github.event.inputs.bump }} - NEW=$(poetry version -s) - echo "old=$OLD" >> $GITHUB_OUTPUT - echo "new=$NEW" >> $GITHUB_OUTPUT - - - name: Update version refs in root pyproject.toml and requirements.txt - run: | - OLD=${{ steps.bump.outputs.old }} - NEW=${{ steps.bump.outputs.new }} - sed -i "s/litellm-enterprise = {version = \"${OLD}\"/litellm-enterprise = {version = \"${NEW}\"/" ../pyproject.toml - sed -i "s/litellm-enterprise==${OLD}/litellm-enterprise==${NEW}/" ../requirements.txt - - - name: Update poetry.lock - working-directory: . - run: poetry lock - - - name: Build - run: poetry build - - - name: Commit version bump and create PR - id: create-pr - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - cd .. - BRANCH="bump/enterprise-${{ steps.bump.outputs.new }}" - git checkout -b "$BRANCH" - git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock - git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" - git push origin "$BRANCH" --force - gh pr create \ - --title "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" \ - --body "Version bump for litellm-enterprise. Merge to update main." \ - --head "$BRANCH" \ - --base main \ - || true - PR_URL=$(gh pr list --head "$BRANCH" --json url -q '.[0].url') - echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT - env: - GH_TOKEN: ${{ github.token }} - - - name: Enable auto-merge - run: | - gh pr merge "${{ steps.create-pr.outputs.pr_url }}" --auto --squash - env: - GH_TOKEN: ${{ github.token }} - - - name: Publish to PyPI - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_ENTERPRISE }} - run: | - pip install twine - twine upload dist/litellm_enterprise-${{ steps.bump.outputs.new }}* diff --git a/.github/workflows/publish_proxy_extras.yml b/.github/workflows/publish_proxy_extras.yml deleted file mode 100644 index fa30b15316..0000000000 --- a/.github/workflows/publish_proxy_extras.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Publish litellm-proxy-extras to PyPI - -on: - workflow_dispatch: - inputs: - bump: - description: "Version bump type" - required: true - default: "patch" - type: choice - options: - - patch - - minor - - major - -jobs: - publish: - runs-on: ubuntu-latest - if: github.repository == 'BerriAI/litellm' - permissions: - contents: write - defaults: - run: - working-directory: litellm-proxy-extras - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Poetry - run: pip install poetry - - - name: Bump version - id: bump - run: | - OLD=$(poetry version -s) - poetry version ${{ github.event.inputs.bump }} - NEW=$(poetry version -s) - echo "old=$OLD" >> $GITHUB_OUTPUT - echo "new=$NEW" >> $GITHUB_OUTPUT - - - name: Update version refs in root pyproject.toml and requirements.txt - run: | - OLD=${{ steps.bump.outputs.old }} - NEW=${{ steps.bump.outputs.new }} - sed -i "s/litellm-proxy-extras = {version = \"${OLD}\"/litellm-proxy-extras = {version = \"${NEW}\"/" ../pyproject.toml - sed -i "s/litellm-proxy-extras==${OLD}/litellm-proxy-extras==${NEW}/" ../requirements.txt - - - name: Update poetry.lock - working-directory: . - run: poetry lock - - - name: Build - run: poetry build - - - name: Commit version bump - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - cd .. - git add litellm-proxy-extras/pyproject.toml pyproject.toml requirements.txt poetry.lock - git commit -m "bump: litellm-proxy-extras ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" - git push - - - name: Publish to PyPI - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }} - run: | - pip install twine - twine upload dist/litellm_proxy_extras-${{ steps.bump.outputs.new }}* diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml new file mode 100644 index 0000000000..8f675bb307 --- /dev/null +++ b/.github/workflows/publish_to_pypi.yml @@ -0,0 +1,136 @@ +name: Publish to PyPI + +on: + workflow_dispatch: + +jobs: + preflight-checks: + name: Preflight Checks + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + # No environment — read-only checks, no approval needed + outputs: + needs_publish: ${{ steps.check-litellm.outputs.needs_publish }} + version: ${{ steps.check-litellm.outputs.version }} + + steps: + - name: Checkout repo + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Check litellm version on PyPI + id: check-litellm + run: | + VERSION=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/') + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Checking if litellm $VERSION exists on PyPI..." + + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm/$VERSION/json") + if [ "$HTTP_STATUS" = "200" ]; then + echo "litellm $VERSION already exists on PyPI. Skipping publish." + echo "needs_publish=false" >> "$GITHUB_OUTPUT" + else + echo "litellm $VERSION not found on PyPI. Publish needed." + echo "needs_publish=true" >> "$GITHUB_OUTPUT" + fi + + - name: Sanity check proxy-extras version + run: | + # Read pinned version from requirements.txt + REQ_VERSION=$(grep -oP 'litellm-proxy-extras==\K[0-9.]+' requirements.txt) + if [ -z "$REQ_VERSION" ]; then + echo "::error::Could not find litellm-proxy-extras version in requirements.txt" + exit 1 + fi + echo "requirements.txt pins litellm-proxy-extras==$REQ_VERSION" + + # Read pinned version from pyproject.toml dependency + PYPROJECT_VERSION=$(python3 -c " + import re + with open('pyproject.toml') as f: + content = f.read() + match = re.search(r'litellm-proxy-extras\s*=\s*\{version\s*=\s*\"([^\"]+)\"', content) + if match: + print(match.group(1).lstrip('^~>=')) + else: + import sys + print('::error::Could not find litellm-proxy-extras dependency in pyproject.toml', file=sys.stderr) + sys.exit(1) + ") + echo "pyproject.toml pins litellm-proxy-extras version: $PYPROJECT_VERSION" + + # Check that both pinned versions match + if [ "$REQ_VERSION" != "$PYPROJECT_VERSION" ]; then + echo "::error::Version mismatch: requirements.txt has $REQ_VERSION but pyproject.toml has $PYPROJECT_VERSION" + exit 1 + fi + + # Check that the pinned version exists on PyPI + echo "Checking if litellm-proxy-extras $REQ_VERSION exists on PyPI..." + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$REQ_VERSION/json") + if [ "$HTTP_STATUS" != "200" ]; then + echo "::error::litellm-proxy-extras $REQ_VERSION is not published on PyPI yet. Publish it before releasing litellm." + exit 1 + fi + echo "litellm-proxy-extras $REQ_VERSION exists on PyPI. Sanity check passed." + + publish-litellm: + name: Publish litellm to PyPI + needs: preflight-checks + if: needs.preflight-checks.outputs.needs_publish == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + id-token: write + contents: read + environment: pypi-publish + + steps: + - name: Checkout repo + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Copy model prices backup + run: cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json + + - name: Install build tools + run: python -m pip install --upgrade pip build==1.4.2 + + - name: Build package + run: | + rm -rf build dist + python -m build + + - name: Verify build artifacts + env: + EXPECTED_VERSION: ${{ needs.preflight-checks.outputs.version }} + run: | + echo "Contents of dist/:" + ls -la dist/ + # Ensure we have both sdist and wheel + ls dist/*.tar.gz + ls dist/*.whl + # Verify built version matches expected + ls dist/ | grep -q "litellm-${EXPECTED_VERSION}" || { + echo "::error::Built artifacts do not match expected version $EXPECTED_VERSION" + ls dist/ + exit 1 + } + + - name: Validate package metadata + run: | + pip install twine==6.2.0 + twine check dist/* + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 diff --git a/.github/workflows/read_pyproject_version.yml b/.github/workflows/read_pyproject_version.yml index 8f6310f935..04b4a38ce1 100644 --- a/.github/workflows/read_pyproject_version.yml +++ b/.github/workflows/read_pyproject_version.yml @@ -3,7 +3,10 @@ name: Read Version from pyproject.toml on: push: branches: - - main # Change this to the default branch of your repository + - main # Change this to the default branch of your repository + +permissions: + contents: read jobs: read-version: @@ -11,20 +14,14 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: - python-version: 3.8 # Adjust the Python version as needed - - - name: Install dependencies - run: pip install toml + persist-credentials: false - name: Read version from pyproject.toml id: read-version run: | - version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])') + version=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/') printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV - name: Display version diff --git a/.github/workflows/redeploy_proxy.py b/.github/workflows/redeploy_proxy.py deleted file mode 100644 index ed46bef73a..0000000000 --- a/.github/workflows/redeploy_proxy.py +++ /dev/null @@ -1,20 +0,0 @@ -""" - -redeploy_proxy.py -""" - -import os -import requests -import time - -# send a get request to this endpoint -deploy_hook1 = os.getenv("LOAD_TEST_REDEPLOY_URL1") -response = requests.get(deploy_hook1, timeout=20) - - -deploy_hook2 = os.getenv("LOAD_TEST_REDEPLOY_URL2") -response = requests.get(deploy_hook2, timeout=20) - -print("SENT GET REQUESTS to re-deploy proxy") -print("sleeeping.... for 60s") -time.sleep(60) diff --git a/.github/workflows/regenerate-poetry-lock.yml b/.github/workflows/regenerate-poetry-lock.yml deleted file mode 100644 index c0844f1c70..0000000000 --- a/.github/workflows/regenerate-poetry-lock.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Regenerate poetry.lock - -# Runs whenever pyproject.toml is merged into main (the most common cause of -# the "pyproject.toml changed significantly since poetry.lock was last generated" -# CI failure). Can also be triggered manually. -on: - push: - branches: - - main - paths: - - pyproject.toml - workflow_dispatch: - -permissions: - contents: write # needed to push the auto/regenerate-poetry-lock-* branch - pull-requests: write # needed to open the PR and enable auto-merge - -jobs: - regenerate-lock: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Poetry - run: pip install poetry - - - name: Regenerate poetry.lock - run: poetry lock - - - name: Check whether poetry.lock actually changed - id: diff - run: | - if git diff --quiet poetry.lock; then - echo "changed=false" >> "$GITHUB_OUTPUT" - else - echo "changed=true" >> "$GITHUB_OUTPUT" - fi - - - name: Open PR with the refreshed lock file - if: steps.diff.outputs.changed == 'true' - id: open-pr - run: | - BRANCH="auto/regenerate-poetry-lock-$(date +'%Y%m%d%H%M%S')" - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git checkout -b "$BRANCH" - git add poetry.lock - git commit -m "chore: regenerate poetry.lock to match pyproject.toml" - git push -f origin "$BRANCH" - - cat > /tmp/pr-body.md << 'BODY' - Automated regeneration of `poetry.lock` after `pyproject.toml` was updated on `main`. - - Fixes the recurring CI failure: - ``` - pyproject.toml changed significantly since poetry.lock was last generated. - Run `poetry lock` to fix the lock file. - ``` - BODY - - PR_URL=$(gh pr create \ - --title "chore: regenerate poetry.lock to match pyproject.toml" \ - --body-file /tmp/pr-body.md \ - --head "$BRANCH" \ - --base main) - echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ github.token }} - - - name: Enable auto-merge - if: steps.diff.outputs.changed == 'true' - run: | - gh pr merge "${{ steps.open-pr.outputs.pr_url }}" --auto --squash - env: - GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/reset_stable.yml b/.github/workflows/reset_stable.yml deleted file mode 100644 index f6fed672d4..0000000000 --- a/.github/workflows/reset_stable.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Reset litellm_stable branch - -on: - release: - types: [published, created] -jobs: - update-stable-branch: - if: ${{ startsWith(github.event.release.tag_name, 'v') && !endsWith(github.event.release.tag_name, '-stable') }} - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Reset litellm_stable_release_branch branch to the release commit - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Configure Git user - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - # Fetch all branches and tags - git fetch --all - - # Check if the litellm_stable_release_branch branch exists - if git show-ref --verify --quiet refs/remotes/origin/litellm_stable_release_branch; then - echo "litellm_stable_release_branch branch exists." - git checkout litellm_stable_release_branch - else - echo "litellm_stable_release_branch branch does not exist. Creating it." - git checkout -b litellm_stable_release_branch - fi - - # Reset litellm_stable_release_branch branch to the release commit - git reset --hard $GITHUB_SHA - - # Push the updated litellm_stable_release_branch branch - git push origin litellm_stable_release_branch --force diff --git a/.github/workflows/run_observatory_tests.yml b/.github/workflows/run_observatory_tests.yml index d343098ed3..a25b96766d 100644 --- a/.github/workflows/run_observatory_tests.yml +++ b/.github/workflows/run_observatory_tests.yml @@ -33,7 +33,9 @@ jobs: timeout-minutes: 30 steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Validate tag input env: @@ -49,11 +51,12 @@ jobs: TAG: ${{ inputs.tag }} AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} + WORKSPACE: ${{ github.workspace }} run: | docker run -d \ --name litellm-rc \ -p 4000:4000 \ - -v "${{ github.workspace }}/.github/observatory/litellm_config.yaml:/app/config.yaml" \ + -v "${WORKSPACE}/.github/observatory/litellm_config.yaml:/app/config.yaml" \ -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \ -e AZURE_API_KEY="${AZURE_API_KEY}" \ -e AZURE_API_BASE="${AZURE_API_BASE}" \ @@ -77,8 +80,9 @@ jobs: - name: Start cloudflared tunnel run: | - # Install cloudflared + # Install cloudflared (pinned version + checksum) curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared + echo "afdfadd1ef552e66bffc35246fe30a9bd578356d2d386de95585ccfc432472b8 /usr/local/bin/cloudflared" | sha256sum -c - chmod +x /usr/local/bin/cloudflared # Start a quick tunnel (no account needed) and capture the URL @@ -103,11 +107,11 @@ jobs: - name: Verify tunnel connectivity run: | - echo "Testing tunnel at ${{ env.TUNNEL_URL }}..." + echo "Testing tunnel at ${TUNNEL_URL}..." # Quick tunnels need time for DNS propagation; retry to avoid # transient NXDOMAIN (curl exit code 6) on first attempt. for i in $(seq 1 10); do - if curl -sf "${{ env.TUNNEL_URL }}/health/liveliness" > /dev/null 2>&1; then + if curl -sf "${TUNNEL_URL}/health/liveliness" > /dev/null 2>&1; then echo "Tunnel is working (attempt $i)" exit 0 fi @@ -221,5 +225,5 @@ jobs: - name: Cleanup if: always() run: | - kill "${{ env.CLOUDFLARED_PID }}" 2>/dev/null || true + kill "$CLOUDFLARED_PID" 2>/dev/null || true docker rm -f litellm-rc 2>/dev/null || true diff --git a/.github/workflows/scan_duplicate_issues.yml b/.github/workflows/scan_duplicate_issues.yml index 06e8f453a8..222ff11f30 100644 --- a/.github/workflows/scan_duplicate_issues.yml +++ b/.github/workflows/scan_duplicate_issues.yml @@ -21,14 +21,15 @@ jobs: contents: read steps: - name: Checkout scripts - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: sparse-checkout: .github/scripts + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: "3.11" + python-version: "3.13" - name: Scan for duplicate issues env: diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 0000000000..7cd12bb219 --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,47 @@ +name: Scorecard supply-chain security + +on: + branch_protection_rule: + schedule: + - cron: '27 12 * * 4' + push: + branches: ["main"] + +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + if: github.event.repository.default_branch == github.ref_name + permissions: + security-events: write + id-token: write + # Uncomment for private repos if needed: + # contents: read + # actions: read + + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + - name: Upload artifact + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + - name: Upload to code scanning + uses: github/codeql-action/upload-sarif@c10b806170c8ee63ea24152429041b5624f0baf5 # v4.35.1 + with: + sarif_file: results.sarif diff --git a/.github/workflows/simple_pypi_publish.yml b/.github/workflows/simple_pypi_publish.yml deleted file mode 100644 index e183055681..0000000000 --- a/.github/workflows/simple_pypi_publish.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Simple PyPI Publish - -on: - workflow_dispatch: - inputs: - version: - description: 'Version to publish (e.g., 1.74.10)' - required: true - type: string - -env: - TWINE_USERNAME: __token__ - -jobs: - publish: - runs-on: ubuntu-latest - if: github.repository == 'BerriAI/litellm' - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install toml build wheel twine - - - name: Update version in pyproject.toml - run: | - python -c " - import toml - - with open('pyproject.toml', 'r') as f: - data = toml.load(f) - - data['tool']['poetry']['version'] = '${{ github.event.inputs.version }}' - - with open('pyproject.toml', 'w') as f: - toml.dump(data, f) - - print(f'Updated version to ${{ github.event.inputs.version }}') - " - - - name: Copy model prices file - run: | - cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json - - - name: Build package - run: | - rm -rf build dist - python -m build - - - name: Publish to PyPI - env: - TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }} - run: | - twine upload dist/* - - - name: Output success - run: | - echo "✅ Successfully published litellm v${{ github.event.inputs.version }} to PyPI" - echo "📦 Package: https://pypi.org/project/litellm/${{ github.event.inputs.version }}/" \ No newline at end of file diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 5a9b19fc9c..c905bb1231 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,19 +2,24 @@ name: "Stale Issue Management" on: schedule: - - cron: '0 0 * * *' # Runs daily at midnight UTC + - cron: "0 0 * * *" # Runs daily at midnight UTC workflow_dispatch: +permissions: + issues: write + pull-requests: write + jobs: stale: + if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest steps: - - uses: actions/stale@v8 + - uses: actions/stale@1160a2240286f5da8ec72b1c0816ce2481aabf84 # v8 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" stale-issue-message: "This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs." stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs." - days-before-stale: 90 # Revert to 60 days - days-before-close: 7 # Revert to 7 days + days-before-stale: 90 # Revert to 60 days + days-before-close: 7 # Revert to 7 days stale-issue-label: "stale" - operations-per-run: 1000 \ No newline at end of file + operations-per-run: 1000 diff --git a/.github/workflows/sync-schema.yml b/.github/workflows/sync-schema.yml new file mode 100644 index 0000000000..72a5c56293 --- /dev/null +++ b/.github/workflows/sync-schema.yml @@ -0,0 +1,73 @@ +name: Sync schema.prisma copies + +on: + pull_request: + paths: + - 'schema.prisma' + +# Scoped to ONLY the permissions needed: +# - contents:write to push the sync commit to the PR branch +# - pull-requests:read is implicit (needed to check out the PR) +permissions: + contents: write + +jobs: + sync: + name: Copy root schema to proxy and proxy-extras + runs-on: ubuntu-latest + timeout-minutes: 5 + # Only run on PRs from branches in THIS repo (not forks). + # Fork PRs cannot push back to the head branch with GITHUB_TOKEN, + # and pull_request events from forks have read-only tokens anyway. + # Also reject PRs from branches named after protected branches to + # prevent pushing directly to main/master. + if: >- + github.event.pull_request.head.repo.full_name == github.repository + && github.head_ref != 'main' + && github.head_ref != 'master' + steps: + - name: Checkout PR branch by SHA + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + # Use the merge commit SHA for safety — github.head_ref is an + # attacker-controlled string (the branch name) and could contain + # unusual characters that cause unexpected git behavior. + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: true # needed for git push + + - name: Reject symlinked schema files + run: | + for f in schema.prisma litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; do + if [ -L "$f" ]; then + echo "::error file=$f::$f is a symlink, which is not allowed" + exit 1 + fi + done + + - name: Copy root schema to other locations + run: | + cp schema.prisma litellm/proxy/schema.prisma + cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma + + - name: Check for changes + id: diff + run: | + if git diff --quiet -- litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "Schemas already in sync. Nothing to do." + else + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "Schema copies need updating." + fi + + - name: Commit synced schemas + if: steps.diff.outputs.changed == 'true' + run: | + # Push to the PR's head branch (need the branch name for git push). + # We checked out by SHA above for safety, so configure the push target explicitly. + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$GITHUB_HEAD_REF" + git add -- litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma + git commit -m "chore: sync schema.prisma copies from root" + git push origin "HEAD:$GITHUB_HEAD_REF" diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index fc0f84a20d..5bb85716a1 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -2,7 +2,10 @@ name: LiteLLM Linting on: pull_request: - branches: [ main ] + branches: [main] + +permissions: + contents: read jobs: lint: @@ -10,69 +13,73 @@ jobs: timeout-minutes: 5 steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - clean: true + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + clean: true + persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Install Poetry - uses: snok/install-poetry@v1 + - name: Install Poetry + run: pip install 'poetry==2.3.2' - - name: Clean Python cache - run: | - find . -type d -name "__pycache__" -exec rm -rf {} + || true - find . -name "*.pyc" -delete || true + - name: Clean Python cache + run: | + find . -type d -name "__pycache__" -exec rm -rf {} + || true + find . -name "*.pyc" -delete || true - - name: Install dependencies - run: | - poetry lock - poetry install --with dev + - name: Check poetry.lock is up to date + run: | + poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1) - - name: Check Black formatting - run: | - cd litellm - poetry run black --check --exclude '/enterprise/' . - cd .. + - name: Install dependencies + run: | + poetry install --with dev - - name: Debug - Check file state - run: | - echo "Current branch:" - git branch --show-current - echo "Last 3 commits:" - git log --oneline -3 - echo "File content around line 43:" - head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10 - - - name: Run Ruff linting - run: | - cd litellm - poetry run ruff check . - cd .. + - name: Check Black formatting + run: | + cd litellm + poetry run black --check --exclude '/enterprise/' . + cd .. - - name: Print OpenAI version - run: | - poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')" + - name: Debug - Check file state + run: | + echo "Current branch:" + git branch --show-current + echo "Last 3 commits:" + git log --oneline -3 + echo "File content around line 43:" + head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10 - - name: Run MyPy type checking - run: | - cd litellm - poetry run mypy . - cd .. + - name: Run Ruff linting + run: | + cd litellm + poetry run ruff check . + cd .. - - name: Check for circular imports - run: | - cd litellm - poetry run python ../tests/documentation_tests/test_circular_imports.py - cd .. + - name: Print OpenAI version + run: | + poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')" - - name: Check import safety - run: | - poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) + - name: Run MyPy type checking + run: | + cd litellm + poetry run mypy . + cd .. + + - name: Check for circular imports + run: | + cd litellm + poetry run python ../tests/documentation_tests/test_circular_imports.py + cd .. + + - name: Check import safety + run: | + poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) secret-scan: runs-on: ubuntu-latest @@ -81,27 +88,28 @@ jobs: contents: read steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Run secret scan test - run: | - pip install pytest - pytest tests/litellm/test_no_hardcoded_secrets.py -v + - name: Run secret scan test + run: | + pip install 'pytest==9.0.2' + pytest tests/litellm/test_no_hardcoded_secrets.py -v - - name: Run ggshield secret scan - env: - GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} - run: | - if [ -n "$GITGUARDIAN_API_KEY" ]; then - pip install ggshield - ggshield secret scan repo . - else - echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan" - fi + - name: Run ggshield secret scan + env: + GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} + run: | + if [ -n "$GITGUARDIAN_API_KEY" ]; then + pip install 'ggshield==1.48.0' + ggshield secret scan repo . + else + echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan" + fi diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml index d0ac28ab41..860d25636c 100644 --- a/.github/workflows/test-litellm-matrix.yml +++ b/.github/workflows/test-litellm-matrix.yml @@ -4,6 +4,9 @@ on: pull_request: branches: [main] +permissions: + contents: read + # Cancel in-progress runs for the same PR concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -12,7 +15,7 @@ concurrency: jobs: test: runs-on: ubuntu-latest - timeout-minutes: 20 # Increased from 15 to 20 + timeout-minutes: 20 # Increased from 15 to 20 strategy: fail-fast: false matrix: @@ -43,7 +46,7 @@ jobs: - name: "integrations" path: "tests/test_litellm/integrations" workers: 2 - reruns: 3 # Integration tests tend to be flakier + reruns: 3 # Integration tests tend to be flakier - name: "core-utils" path: "tests/test_litellm/litellm_core_utils" workers: 2 @@ -117,18 +120,20 @@ jobs: name: test (${{ matrix.test-group.name }}) steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install Poetry - uses: snok/install-poetry@v1 + run: pip install 'poetry==2.3.2' - name: Cache Poetry dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 with: path: | ~/.cache/pypoetry @@ -144,14 +149,17 @@ jobs: poetry install --with dev,proxy-dev --extras "proxy semantic-router" # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies poetry run pip install google-genai==1.22.0 \ - google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - name: Setup litellm-enterprise run: | poetry run pip install --force-reinstall --no-deps -e enterprise/ - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 poetry run prisma generate --schema litellm/proxy/schema.prisma - name: Run tests - ${{ matrix.test-group.name }} diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml index b0a8b648a4..6b0b3a413a 100644 --- a/.github/workflows/test-litellm-ui-build.yml +++ b/.github/workflows/test-litellm-ui-build.yml @@ -16,10 +16,12 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0 with: node-version: "20" cache: "npm" diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index 3f8369df92..0c040b3ebe 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -4,45 +4,50 @@ name: LiteLLM Mock Tests (folder - tests/test_litellm) # the same tests in parallel across 10 jobs for faster CI times. # Kept for manual debugging only. on: - workflow_dispatch: # Manual trigger only + workflow_dispatch: # Manual trigger only # pull_request: # branches: [ main ] +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest timeout-minutes: 25 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - - name: Thank You Message - run: | - echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY - echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY + - name: Thank You Message + run: | + echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY + echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Install Poetry - uses: snok/install-poetry@v1 + - name: Install Poetry + run: pip install 'poetry==2.3.2' - - name: Install dependencies - run: | - poetry lock - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install "pytest-retry==1.6.3" - poetry run pip install pytest-xdist - poetry run pip install "google-genai==1.22.0" - poetry run pip install "google-cloud-aiplatform>=1.38" - poetry run pip install "fastapi-offline==1.7.3" - poetry run pip install "python-multipart>=0.0.20" - poetry run pip install "openapi-core" - - name: Setup litellm-enterprise as local package - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ - - name: Run tests - run: | - poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 + - name: Install dependencies + run: | + poetry lock + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install "pytest-retry==1.6.3" + poetry run pip install 'pytest-xdist==3.8.0' + poetry run pip install "google-genai==1.22.0" + poetry run pip install "google-cloud-aiplatform==1.115.0" + poetry run pip install "fastapi-offline==1.7.3" + poetry run pip install "python-multipart==0.0.22" + poetry run pip install "openapi-core==0.23.0" + - name: Setup litellm-enterprise as local package + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + - name: Run tests + run: | + poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml index 2e32aae768..1b228ab76b 100644 --- a/.github/workflows/test-mcp.yml +++ b/.github/workflows/test-mcp.yml @@ -2,7 +2,10 @@ name: LiteLLM MCP Tests (folder - tests/mcp_tests) on: pull_request: - branches: [ main ] + branches: [main] + +permissions: + contents: read jobs: test: @@ -10,38 +13,40 @@ jobs: timeout-minutes: 25 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - - name: Thank You Message - run: | - echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY - echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY + - name: Thank You Message + run: | + echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY + echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" - - name: Install Poetry - uses: snok/install-poetry@v1 + - name: Install Poetry + run: pip install 'poetry==2.3.2' - - name: Install dependencies - run: | - poetry lock - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install "pytest==7.3.1" - poetry run pip install "pytest-retry==1.6.3" - poetry run pip install "pytest-cov==5.0.0" - poetry run pip install "pytest-asyncio==0.21.1" - poetry run pip install "respx==0.22.0" - poetry run pip install "pydantic==2.11.0" - poetry run pip install "mcp==1.25.0" - poetry run pip install pytest-xdist + - name: Install dependencies + run: | + poetry lock + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install "pytest==7.3.1" + poetry run pip install "pytest-retry==1.6.3" + poetry run pip install "pytest-cov==5.0.0" + poetry run pip install "pytest-asyncio==0.21.1" + poetry run pip install "respx==0.22.0" + poetry run pip install "pydantic==2.11.0" + poetry run pip install "mcp==1.25.0" + poetry run pip install 'pytest-xdist==3.8.0' - - name: Setup litellm-enterprise as local package - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ + - name: Setup litellm-enterprise as local package + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ - - name: Run MCP tests - run: | - poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5 + - name: Run MCP tests + run: | + poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5 diff --git a/.github/workflows/test-model-map.yaml b/.github/workflows/test-model-map.yaml index ae5ac402e2..429f9e1ce0 100644 --- a/.github/workflows/test-model-map.yaml +++ b/.github/workflows/test-model-map.yaml @@ -2,13 +2,18 @@ name: Validate model_prices_and_context_window.json on: pull_request: - branches: [ main ] + branches: [main] + +permissions: + contents: read jobs: validate-model-prices-json: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Validate model_prices_and_context_window.json run: | diff --git a/.github/workflows/test-proxy-e2e-azure-batches.yml b/.github/workflows/test-proxy-e2e-azure-batches.yml index 4d74f3db0a..7cbbe0b338 100644 --- a/.github/workflows/test-proxy-e2e-azure-batches.yml +++ b/.github/workflows/test-proxy-e2e-azure-batches.yml @@ -9,6 +9,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: proxy_e2e_azure_batches_tests: runs-on: ubuntu-latest @@ -30,18 +33,20 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install Poetry - uses: snok/install-poetry@v1 + run: pip install 'poetry==2.3.2' - name: Cache Poetry dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 with: path: | ~/.cache/pypoetry @@ -56,14 +61,17 @@ jobs: run: | poetry config virtualenvs.in-project true poetry install --with dev,proxy-dev --extras "proxy" - poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity + poetry run pip install psycopg2-binary==2.9.11 uvicorn==0.42.0 fastapi==0.135.2 httpx==0.28.1 tenacity==9.1.4 - name: Setup litellm-enterprise run: | poetry run pip install --force-reinstall --no-deps -e enterprise/ - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 poetry run prisma generate --schema litellm/proxy/schema.prisma - name: Run Prisma migrations @@ -87,4 +95,3 @@ jobs: --tb=short \ --maxfail=3 \ --durations=10 - diff --git a/.github/workflows/test-unit-caching-redis.yml b/.github/workflows/test-unit-caching-redis.yml new file mode 100644 index 0000000000..ca274324f2 --- /dev/null +++ b/.github/workflows/test-unit-caching-redis.yml @@ -0,0 +1,38 @@ +name: "Unit Tests: Caching (Redis)" + +# Uses cloud Redis credentials — only runs on trusted branches, not PRs. +# This prevents external PRs from accessing Redis credentials. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + caching-redis: + uses: ./.github/workflows/_test-unit-services-base.yml + with: + # Redis-only tests that do NOT require provider API keys. + # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py, + # test_router_caching.py) are in Phase 3 integration workflows. + test-path: >- + tests/local_testing/test_dual_cache.py + tests/local_testing/test_redis_batch_optimizations.py + tests/local_testing/test_router_utils.py + workers: 2 + reruns: 2 + timeout-minutes: 20 + enable-redis: true + enable-postgres: false + secrets: + REDIS_HOST: ${{ secrets.REDIS_HOST }} + REDIS_PORT: ${{ secrets.REDIS_PORT }} + REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test-unit-core-utils.yml b/.github/workflows/test-unit-core-utils.yml new file mode 100644 index 0000000000..2f3698fdf6 --- /dev/null +++ b/.github/workflows/test-unit-core-utils.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Core Utilities" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + core-utils: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/litellm_core_utils" + workers: 2 + reruns: 1 diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml new file mode 100644 index 0000000000..d8b30de684 --- /dev/null +++ b/.github/workflows/test-unit-documentation.yml @@ -0,0 +1,67 @@ +name: "Unit Tests: Documentation Validation" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + documentation: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + # Run the same documentation tests that CircleCI ran (as direct Python scripts) + - name: Run documentation validation tests + run: | + poetry run python ./tests/documentation_tests/test_env_keys.py + poetry run python ./tests/documentation_tests/test_router_settings.py + poetry run python ./tests/documentation_tests/test_api_docs.py + poetry run python ./tests/documentation_tests/test_circular_imports.py diff --git a/.github/workflows/test-unit-enterprise-routing.yml b/.github/workflows/test-unit-enterprise-routing.yml new file mode 100644 index 0000000000..13ae3efedb --- /dev/null +++ b/.github/workflows/test-unit-enterprise-routing.yml @@ -0,0 +1,24 @@ +name: "Unit Tests: Enterprise, Google GenAI & Routing" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + enterprise-routing: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/enterprise + tests/test_litellm/google_genai + tests/test_litellm/router_utils + tests/test_litellm/router_strategy + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-integrations.yml b/.github/workflows/test-unit-integrations.yml new file mode 100644 index 0000000000..2789f99d81 --- /dev/null +++ b/.github/workflows/test-unit-integrations.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Integrations (Callbacks & Logging)" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + integrations: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/integrations" + workers: 2 + reruns: 3 diff --git a/.github/workflows/test-unit-llm-providers.yml b/.github/workflows/test-unit-llm-providers.yml new file mode 100644 index 0000000000..6c00272b0c --- /dev/null +++ b/.github/workflows/test-unit-llm-providers.yml @@ -0,0 +1,29 @@ +name: "Unit Tests: LLM Provider Transformations" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + vertex-ai: + name: Vertex AI + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/llms/vertex_ai" + workers: 1 + reruns: 2 + + other-providers: + name: All Other Providers + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml new file mode 100644 index 0000000000..9228decd7c --- /dev/null +++ b/.github/workflows/test-unit-misc.yml @@ -0,0 +1,31 @@ +name: "Unit Tests: MCP, Secrets, Containers & Misc" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + misc: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/secret_managers + tests/test_litellm/a2a_protocol + tests/test_litellm/anthropic_interface + tests/test_litellm/completion_extras + tests/test_litellm/containers + tests/test_litellm/experimental_mcp_client + tests/test_litellm/images + tests/test_litellm/interactions + tests/test_litellm/passthrough + tests/test_litellm/vector_stores + tests/test_litellm/test_*.py + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-auth.yml b/.github/workflows/test-unit-proxy-auth.yml new file mode 100644 index 0000000000..e71821db70 --- /dev/null +++ b/.github/workflows/test-unit-proxy-auth.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Proxy Auth & Key Management" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-auth: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml new file mode 100644 index 0000000000..bdfb6efeef --- /dev/null +++ b/.github/workflows/test-unit-proxy-db.yml @@ -0,0 +1,45 @@ +name: "Unit Tests: Proxy DB Operations" + +# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + proxy-db: + strategy: + fail-fast: false + matrix: + include: + # Key generation tests must NOT run in parallel (event loop conflicts with logging worker) + - test-group: key-generation + test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py" + workers: 0 + timeout: 30 + - test-group: auth-checks + test-path: "tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py" + workers: 8 + timeout: 20 + - test-group: remaining + test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py" + workers: 8 + timeout: 20 + uses: ./.github/workflows/_test-unit-services-base.yml + with: + test-path: ${{ matrix.test-path }} + workers: ${{ matrix.workers }} + reruns: 2 + timeout-minutes: ${{ matrix.timeout }} + enable-redis: false + enable-postgres: true + secrets: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml new file mode 100644 index 0000000000..caff3b3ae0 --- /dev/null +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -0,0 +1,35 @@ +name: "Unit Tests: Proxy API Endpoints" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-endpoints: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/proxy/management_endpoints + tests/test_litellm/proxy/guardrails + tests/test_litellm/proxy/management_helpers + tests/test_litellm/proxy/anthropic_endpoints + tests/test_litellm/proxy/google_endpoints + tests/test_litellm/proxy/openai_files_endpoint + tests/test_litellm/proxy/response_api_endpoints + tests/test_litellm/proxy/image_endpoints + tests/test_litellm/proxy/vector_store_endpoints + tests/test_litellm/proxy/agent_endpoints + tests/test_litellm/proxy/discovery_endpoints + tests/test_litellm/proxy/health_endpoints + tests/test_litellm/proxy/public_endpoints + tests/test_litellm/proxy/prompts + tests/test_litellm/proxy/ui_crud_endpoints + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-infra.yml b/.github/workflows/test-unit-proxy-infra.yml new file mode 100644 index 0000000000..4dfbbe317e --- /dev/null +++ b/.github/workflows/test-unit-proxy-infra.yml @@ -0,0 +1,28 @@ +name: "Unit Tests: Proxy Infrastructure" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-infra: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/proxy/db + tests/test_litellm/proxy/middleware + tests/test_litellm/proxy/spend_tracking + tests/test_litellm/proxy/pass_through_endpoints + tests/test_litellm/proxy/_experimental + tests/test_litellm/proxy/experimental + tests/test_litellm/proxy/common_utils + tests/test_litellm/proxy/test_*.py + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-legacy.yml b/.github/workflows/test-unit-proxy-legacy.yml new file mode 100644 index 0000000000..a939113726 --- /dev/null +++ b/.github/workflows/test-unit-proxy-legacy.yml @@ -0,0 +1,96 @@ +name: "Unit Tests: Proxy Legacy Tests" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + test-group: + - name: "auth-and-jwt" + path: "tests/proxy_unit_tests/test_[a-j]*.py" + - name: "key-generation" + path: "tests/proxy_unit_tests/test_[k-o]*.py" + - name: "proxy-config" + path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py" + - name: "proxy-server" + path: "tests/proxy_unit_tests/test_proxy_server.py" + - name: "proxy-server-extras" + path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py" + - name: "proxy-utils" + path: "tests/proxy_unit_tests/test_proxy_utils.py" + - name: "proxy-token-counter" + path: "tests/proxy_unit_tests/test_proxy_token_counter.py" + - name: "proxy-response-and-misc" + path: "tests/proxy_unit_tests/test_[r-t]*.py" + - name: "proxy-user-auth-and-spend" + path: "tests/proxy_unit_tests/test_[u-z]*.py" + + name: ${{ matrix.test-group.name }} + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run tests - ${{ matrix.test-group.name }} + env: + TEST_PATH: ${{ matrix.test-group.path }} + run: | + poetry run pytest ${TEST_PATH} \ + --tb=short -vv \ + --maxfail=10 \ + -n 2 \ + --reruns 1 \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 diff --git a/.github/workflows/test-unit-responses-caching-types.yml b/.github/workflows/test-unit-responses-caching-types.yml new file mode 100644 index 0000000000..7f3acac280 --- /dev/null +++ b/.github/workflows/test-unit-responses-caching-types.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Responses, Caching & Types" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + responses-caching-types: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-security.yml b/.github/workflows/test-unit-security.yml new file mode 100644 index 0000000000..b38c82b1c2 --- /dev/null +++ b/.github/workflows/test-unit-security.yml @@ -0,0 +1,28 @@ +name: "Unit Tests: Security" + +# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + security: + uses: ./.github/workflows/_test-unit-services-base.yml + with: + test-path: "tests/proxy_security_tests/" + workers: 1 + reruns: 2 + timeout-minutes: 20 + enable-redis: false + enable-postgres: true + secrets: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test_server_root_path.yml b/.github/workflows/test_server_root_path.yml index c359e38bff..47636ce8e9 100644 --- a/.github/workflows/test_server_root_path.yml +++ b/.github/workflows/test_server_root_path.yml @@ -17,13 +17,15 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12 - name: Build Docker image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 #v6.14 with: context: . file: ./docker/Dockerfile.non_root diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml new file mode 100644 index 0000000000..9a1e899fed --- /dev/null +++ b/.github/workflows/zizmor.yml @@ -0,0 +1,31 @@ +name: GitHub Actions Security Analysis + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: {} + +jobs: + zizmor: + name: zizmor + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + security-events: write + contents: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Run zizmor + uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9396f323e4..2bc361bc48 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,12 +14,12 @@ repos: types: [python] files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py exclude: ^litellm/__init__.py$ - # - id: black - # name: black - # entry: poetry run black - # language: system - # types: [python] - # files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py + - id: black + name: black + entry: poetry run black + language: system + types: [python] + files: (litellm/|litellm_proxy_extras/).*\.py - repo: https://github.com/pycqa/flake8 rev: 7.0.0 # The version of flake8 to use hooks: diff --git a/.semgrep/rules/security/no-claude-directory.yml b/.semgrep/rules/security/no-claude-directory.yml new file mode 100644 index 0000000000..7d120a7c23 --- /dev/null +++ b/.semgrep/rules/security/no-claude-directory.yml @@ -0,0 +1,18 @@ +rules: + - id: no-claude-directory-committed + message: > + .claude/ directory must not be committed to the repository. + It contains local Claude Code settings (permissions, worktree paths) that are + developer-machine-specific and may expose internal paths or credentials. + Add .claude/ to .gitignore instead. + severity: ERROR + languages: [generic] + paths: + include: + - "/.claude/**" + - "/.claude/*" + pattern-regex: '[\s\S]+' + metadata: + category: security + tags: [supply-chain, secrets] + confidence: HIGH diff --git a/README.md b/README.md index 67f2f3a204..a6dc597574 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature