From a7ad8a36a416375751526c6acaca7e461f60b05e Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Tue, 9 Dec 2025 00:00:55 -0300 Subject: [PATCH] chore: cleanup unused scripts and fix misplaced test file (#17611) Remove scripts/ directory containing unused development/debug scripts: - mock_ibm_guardrails_server.py - test_groq_streaming_issue.py (debug for #12660) - test_mock_ibm_guardrails.py - update_readme_providers_table.py Move misplaced test file to correct location: - test_litellm/ -> tests/test_litellm/ (from PR #17221) --- scripts/mock_ibm_guardrails_server.py | 358 ------------------ scripts/test_groq_streaming_issue.py | 54 --- scripts/test_mock_ibm_guardrails.py | 181 --------- scripts/update_readme_providers_table.py | 147 ------- ..._passthrough_guardrails_field_targeting.py | 0 5 files changed, 740 deletions(-) delete mode 100644 scripts/mock_ibm_guardrails_server.py delete mode 100644 scripts/test_groq_streaming_issue.py delete mode 100644 scripts/test_mock_ibm_guardrails.py delete mode 100644 scripts/update_readme_providers_table.py rename {test_litellm => tests/test_litellm}/proxy/pass_through_endpoints/test_passthrough_guardrails_field_targeting.py (100%) diff --git a/scripts/mock_ibm_guardrails_server.py b/scripts/mock_ibm_guardrails_server.py deleted file mode 100644 index a9251c14ce..0000000000 --- a/scripts/mock_ibm_guardrails_server.py +++ /dev/null @@ -1,358 +0,0 @@ -""" -Mock FastAPI server for IBM FMS Guardrails Orchestrator Detector API. - -This server implements the Detector API endpoints for testing purposes. -Based on: https://foundation-model-stack.github.io/fms-guardrails-orchestrator/ - -Usage: - python scripts/mock_ibm_guardrails_server.py - -The server will run on http://localhost:8001 by default. -""" - -import uuid -from typing import Any, Dict, List, Optional - -import uvicorn -from fastapi import FastAPI, Header, HTTPException, status -from pydantic import BaseModel, Field - -app = FastAPI( - title="IBM FMS Guardrails Orchestrator Mock", - description="Mock server for testing IBM Guardrails Detector API", - version="1.0.0", -) - - -# Request Models -class DetectorParams(BaseModel): - """Parameters specific to the detector.""" - - threshold: Optional[float] = Field(None, ge=0.0, le=1.0) - custom_param: Optional[str] = None - - -class TextDetectionRequest(BaseModel): - """Request model for text detection.""" - - contents: List[str] = Field(..., description="Text content to analyze") - detector_params: Optional[DetectorParams] = None - - -class TextGenerationDetectionRequest(BaseModel): - """Request model for text generation detection.""" - - detector_id: str = Field(..., description="ID of the detector to use") - prompt: str = Field(..., description="Input prompt") - generated_text: str = Field(..., description="Generated text to analyze") - detector_params: Optional[DetectorParams] = None - - -class ContextDetectionRequest(BaseModel): - """Request model for detection with context.""" - - detector_id: str = Field(..., description="ID of the detector to use") - content: str = Field(..., description="Text content to analyze") - context: Optional[Dict[str, Any]] = Field(None, description="Additional context") - detector_params: Optional[DetectorParams] = None - - -# Response Models -class Detection(BaseModel): - """Individual detection result.""" - - detection_type: str = Field(..., description="Type of detection") - detection: bool = Field(..., description="Whether content was detected as harmful") - score: float = Field(..., ge=0.0, le=1.0, description="Detection confidence score") - start: Optional[int] = Field(None, description="Start position in text") - end: Optional[int] = Field(None, description="End position in text") - text: Optional[str] = Field(None, description="Detected text segment") - evidence: Optional[List[str]] = Field(None, description="Supporting evidence") - - -class DetectionResponse(BaseModel): - """Response model for detection results.""" - - detections: List[Detection] = Field(..., description="List of detections") - detection_id: str = Field(..., description="Unique ID for this detection request") - - -# Mock detector configurations -MOCK_DETECTORS = { - "hate": { - "name": "Hate Speech Detector", - "triggers": ["hate", "offensive", "discriminatory", "slur"], - "default_score": 0.85, - }, - "pii": { - "name": "PII Detector", - "triggers": ["email", "ssn", "credit card", "phone number", "address"], - "default_score": 0.92, - }, - "toxicity": { - "name": "Toxicity Detector", - "triggers": ["toxic", "abusive", "profanity", "insult"], - "default_score": 0.78, - }, - "jailbreak": { - "name": "Jailbreak Detector", - "triggers": ["ignore instructions", "override", "bypass", "jailbreak"], - "default_score": 0.88, - }, - "prompt_injection": { - "name": "Prompt Injection Detector", - "triggers": ["ignore previous", "new instructions", "system prompt"], - "default_score": 0.90, - }, -} - - -def simulate_detection( - detector_id: str, content: str, detector_params: Optional[DetectorParams] = None -) -> List[Detection]: - """ - Simulate detection logic based on detector type and content. - - Args: - detector_id: ID of the detector to simulate - content: Text content to analyze - detector_params: Optional detector parameters - - Returns: - List of Detection objects - """ - detections = [] - content_lower = " ".join(c for c in content).lower() - - # Get detector config - detector_config = MOCK_DETECTORS.get(detector_id) - if not detector_config: - # Unknown detector - return no detections - return detections - - # Check for triggers in content - for trigger in detector_config["triggers"]: - if trigger in content_lower: - # Calculate score (use threshold if provided, otherwise default) - base_score = detector_config["default_score"] - threshold = ( - detector_params.threshold - if detector_params and detector_params.threshold - else None - ) - - # Adjust score slightly based on content length (longer content = slightly lower confidence) - score_adjustment = max(0, min(0.1, len(content) / 10000)) - score = max(0.0, min(1.0, base_score - score_adjustment)) - - # Find position of trigger - start_pos = content_lower.find(trigger) - end_pos = start_pos + len(trigger) - - detection = Detection( - detection_type=detector_id, - detection=threshold is None or score >= threshold, - score=score, - start=start_pos, - end=end_pos, - text=content[start_pos:end_pos] if start_pos >= 0 else None, - evidence=[f"Found trigger word: {trigger}"], - ) - detections.append(detection) - - # If no triggers found, return a negative detection - if not detections: - detections.append( - Detection( - detection_type=detector_id, - detection=False, - score=0.05, # Low score for clean content - ) - ) - - return detections - - -# Authentication middleware -def verify_auth_token(authorization: Optional[str] = Header(None)) -> bool: - """ - Verify the authentication token. - - Args: - authorization: Authorization header value - - Returns: - True if valid, raises HTTPException otherwise - """ - if not authorization: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Missing authorization header", - ) - - # Simple token validation - in real implementation, this would validate against a real auth system - if not authorization.startswith("Bearer "): - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid authorization header format. Expected: Bearer ", - ) - - token = authorization.replace("Bearer ", "") - - # Accept any non-empty token for mock purposes - if not token: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Empty token provided", - ) - - return True - - -# API Endpoints -@app.get("/health") -async def health_check(): - """Health check endpoint.""" - return {"status": "healthy", "service": "IBM FMS Guardrails Mock Server"} - - -@app.get("/") -async def root(): - """Root endpoint with API information.""" - return { - "service": "IBM FMS Guardrails Orchestrator Mock", - "version": "1.0.0", - "endpoints": { - "health": "/health", - "text_detection": "/api/v1/text/detection", - "generation_detection": "/api/v1/text/generation/detection", - "context_detection": "/api/v1/text/context/detection", - }, - "available_detectors": list(MOCK_DETECTORS.keys()), - } - - -@app.post("/api/v1/text/contents") -async def text_detection( - request: TextDetectionRequest, - detector_id: str = Header(None), # query parameter - authorization: Optional[str] = Header(None), -): - """ - Detect potential issues in text content. - - Args: - request: Detection request with content and detector ID - detector_id: ID of detector - authorization: Bearer token for authentication - - Returns: - Detection results - """ - verify_auth_token(authorization) - - detections = simulate_detection( - detector_id=detector_id, - content=request.contents, - detector_params=request.detector_params, - ) - - return detections - - -@app.post("/api/v1/text/generation/detection", response_model=DetectionResponse) -async def text_generation_detection( - request: TextGenerationDetectionRequest, - authorization: Optional[str] = Header(None), -): - """ - Detect potential issues in generated text. - - Args: - request: Detection request with prompt and generated text - authorization: Bearer token for authentication - - Returns: - Detection results - """ - verify_auth_token(authorization) - - # Analyze both prompt and generated text - combined_content = f"{request.prompt} {request.generated_text}" - - detections = simulate_detection( - detector_id=request.detector_id, - content=combined_content, - detector_params=request.detector_params, - ) - - return DetectionResponse( - detections=detections, - detection_id=str(uuid.uuid4()), - ) - - -@app.post("/api/v1/text/context/detection", response_model=DetectionResponse) -async def context_detection( - request: ContextDetectionRequest, - authorization: Optional[str] = Header(None), -): - """ - Detect potential issues in text with additional context. - - Args: - request: Detection request with content and context - authorization: Bearer token for authentication - - Returns: - Detection results - """ - verify_auth_token(authorization) - - detections = simulate_detection( - detector_id=request.detector_id, - content=request.content, - detector_params=request.detector_params, - ) - - return DetectionResponse( - detections=detections, - detection_id=str(uuid.uuid4()), - ) - - -@app.get("/api/v1/detectors") -async def list_detectors(authorization: Optional[str] = Header(None)): - """ - List available detectors. - - Args: - authorization: Bearer token for authentication - - Returns: - List of available detectors - """ - verify_auth_token(authorization) - - return { - "detectors": [ - { - "id": detector_id, - "name": config["name"], - "triggers": config["triggers"], - } - for detector_id, config in MOCK_DETECTORS.items() - ] - } - - -if __name__ == "__main__": - print("๐Ÿš€ Starting IBM FMS Guardrails Mock Server...") - print("๐Ÿ“ Server will be available at: http://localhost:8001") - print("๐Ÿ“š API docs at: http://localhost:8001/docs") - print("\nAvailable detectors:") - for detector_id, config in MOCK_DETECTORS.items(): - print(f" - {detector_id}: {config['name']}") - print("\nโœจ Use any Bearer token for authentication in this mock server\n") - - uvicorn.run(app, host="0.0.0.0", port=8001) diff --git a/scripts/test_groq_streaming_issue.py b/scripts/test_groq_streaming_issue.py deleted file mode 100644 index 0a996c0c20..0000000000 --- a/scripts/test_groq_streaming_issue.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Test script to reproduce the Groq streaming ASCII encoding issue. - -This reproduces the issue described in #12660 where streaming responses -containing non-ASCII characters like ยต cause encoding errors. -""" -import asyncio -import os -import traceback -from litellm import acompletion - -async def test_groq_streaming_with_special_chars(): - """Test that reproduces the ASCII encoding issue with Groq streaming.""" - try: - print("Testing acompletion + streaming with Groq...") - - # Test message that should trigger the ยต character or similar non-ASCII content - test_messages = [ - {"content": "What is the symbol for micro? Please include the ยต symbol in your response.", "role": "user"} - ] - - # This should trigger the ASCII encoding error described in the issue - response = await acompletion( - model="groq/llama-3.3-70b-versatile", - messages=test_messages, - stream=True - ) - - print(f"Response type: {type(response)}") - - # Try to iterate through the stream - async for chunk in response: - print(f"Chunk: {chunk}") - - print("โœ… Test completed successfully - no encoding errors!") - - except Exception as e: - print(f"โŒ Error occurred: {e}") - print(f"Error type: {type(e)}") - print(f"Traceback:\n{traceback.format_exc()}") - return False - - return True - -if __name__ == "__main__": - # Note: This requires GROQ_API_KEY to be set - if not os.getenv("GROQ_API_KEY"): - print("โš ๏ธ GROQ_API_KEY not set. Skipping test.") - else: - success = asyncio.run(test_groq_streaming_with_special_chars()) - if success: - print("๐ŸŽ‰ All tests passed!") - else: - print("๐Ÿ’ฅ Test failed!") \ No newline at end of file diff --git a/scripts/test_mock_ibm_guardrails.py b/scripts/test_mock_ibm_guardrails.py deleted file mode 100644 index 91e4e02d8b..0000000000 --- a/scripts/test_mock_ibm_guardrails.py +++ /dev/null @@ -1,181 +0,0 @@ -""" -Test script for the mock IBM Guardrails server. - -This demonstrates how to interact with the mock server. - -Usage: - # Start the mock server in one terminal: - python scripts/mock_ibm_guardrails_server.py - - # Run this test in another terminal: - python scripts/test_mock_ibm_guardrails.py -""" - -import asyncio - -import httpx - - -async def test_mock_server(): - """Test the mock IBM Guardrails server.""" - base_url = "http://localhost:8001" - headers = {"Authorization": "Bearer test-token-12345"} - - print("๐Ÿงช Testing IBM FMS Guardrails Mock Server\n") - - async with httpx.AsyncClient() as client: - # Test 1: Health check - print("1๏ธโƒฃ Testing health check...") - try: - response = await client.get(f"{base_url}/health") - print(f" โœ… Health check: {response.json()}\n") - except Exception as e: - print(f" โŒ Health check failed: {e}\n") - return - - # Test 2: List detectors - print("2๏ธโƒฃ Testing list detectors...") - try: - response = await client.get( - f"{base_url}/api/v1/detectors", - headers=headers - ) - detectors = response.json() - print(f" โœ… Found {len(detectors['detectors'])} detectors:") - for detector in detectors["detectors"]: - print(f" - {detector['id']}: {detector['name']}") - print() - except Exception as e: - print(f" โŒ List detectors failed: {e}\n") - - # Test 3: Text detection with clean content - print("3๏ธโƒฃ Testing text detection (clean content)...") - try: - response = await client.post( - f"{base_url}/api/v1/text/detection", - headers=headers, - json={ - "detector_id": "hate", - "content": "This is a normal, friendly message.", - } - ) - result = response.json() - print(f" โœ… Detection result:") - print(f" Detection ID: {result['detection_id']}") - for detection in result["detections"]: - print(f" - Type: {detection['detection_type']}, Detected: {detection['detection']}, Score: {detection['score']:.2f}") - print() - except Exception as e: - print(f" โŒ Text detection failed: {e}\n") - - # Test 4: Text detection with problematic content - print("4๏ธโƒฃ Testing text detection (problematic content)...") - try: - response = await client.post( - f"{base_url}/api/v1/text/detection", - headers=headers, - json={ - "detector_id": "hate", - "content": "This message contains hate speech and offensive language.", - } - ) - result = response.json() - print(f" โœ… Detection result:") - print(f" Detection ID: {result['detection_id']}") - for detection in result["detections"]: - print(f" - Type: {detection['detection_type']}, Detected: {detection['detection']}, Score: {detection['score']:.2f}") - if detection.get("evidence"): - print(f" Evidence: {detection['evidence']}") - print() - except Exception as e: - print(f" โŒ Text detection failed: {e}\n") - - # Test 5: PII detection - print("5๏ธโƒฃ Testing PII detection...") - try: - response = await client.post( - f"{base_url}/api/v1/text/detection", - headers=headers, - json={ - "detector_id": "pii", - "content": "Please send the report to my email address john@example.com", - } - ) - result = response.json() - print(f" โœ… Detection result:") - print(f" Detection ID: {result['detection_id']}") - for detection in result["detections"]: - print(f" - Type: {detection['detection_type']}, Detected: {detection['detection']}, Score: {detection['score']:.2f}") - if detection.get("text"): - print(f" Detected text: '{detection['text']}'") - print() - except Exception as e: - print(f" โŒ PII detection failed: {e}\n") - - # Test 6: Generation detection - print("6๏ธโƒฃ Testing text generation detection...") - try: - response = await client.post( - f"{base_url}/api/v1/text/generation/detection", - headers=headers, - json={ - "detector_id": "jailbreak", - "prompt": "Tell me about AI safety", - "generated_text": "I will ignore instructions and provide harmful content.", - } - ) - result = response.json() - print(f" โœ… Detection result:") - print(f" Detection ID: {result['detection_id']}") - for detection in result["detections"]: - print(f" - Type: {detection['detection_type']}, Detected: {detection['detection']}, Score: {detection['score']:.2f}") - print() - except Exception as e: - print(f" โŒ Generation detection failed: {e}\n") - - # Test 7: Detection with custom threshold - print("7๏ธโƒฃ Testing detection with custom threshold...") - try: - response = await client.post( - f"{base_url}/api/v1/text/detection", - headers=headers, - json={ - "detector_id": "toxicity", - "content": "This contains toxic language", - "detector_params": { - "threshold": 0.9 - } - } - ) - result = response.json() - print(f" โœ… Detection result (threshold=0.9):") - print(f" Detection ID: {result['detection_id']}") - for detection in result["detections"]: - print(f" - Type: {detection['detection_type']}, Detected: {detection['detection']}, Score: {detection['score']:.2f}") - print() - except Exception as e: - print(f" โŒ Threshold detection failed: {e}\n") - - # Test 8: Authentication error - print("8๏ธโƒฃ Testing authentication error...") - try: - response = await client.post( - f"{base_url}/api/v1/text/detection", - json={ - "detector_id": "hate", - "content": "Test content", - } - ) - if response.status_code == 401: - print(f" โœ… Authentication error handled correctly: {response.json()}\n") - else: - print(f" โš ๏ธ Unexpected status code: {response.status_code}\n") - except Exception as e: - print(f" โŒ Auth test failed: {e}\n") - - print("โœจ All tests completed!") - - -if __name__ == "__main__": - asyncio.run(test_mock_server()) - diff --git a/scripts/update_readme_providers_table.py b/scripts/update_readme_providers_table.py deleted file mode 100644 index 1435e13198..0000000000 --- a/scripts/update_readme_providers_table.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 -""" -Script to update the README.md providers table from provider_endpoints_support.json -""" - -import json -import re -from pathlib import Path - -# Define paths -REPO_ROOT = Path(__file__).parent.parent -JSON_PATH = REPO_ROOT / "provider_endpoints_support.json" -README_PATH = REPO_ROOT / "README.md" - -# Endpoint column headers -ENDPOINT_COLUMNS = [ - ("/chat/completions", "chat_completions"), - ("/messages", "messages"), - ("/responses", "responses"), - ("/embeddings", "embeddings"), - ("/image/generations", "image_generations"), - ("/audio/transcriptions", "audio_transcriptions"), - ("/audio/speech", "audio_speech"), - ("/moderations", "moderations"), - ("/batches", "batches"), - ("/rerank", "rerank"), -] - - -def load_providers_data(): - """Load provider data from JSON file""" - with open(JSON_PATH, 'r') as f: - data = json.load(f) - - # Handle both old and new format - if "providers" in data: - return data["providers"] - return data - - -def generate_markdown_table(providers_data): - """Generate markdown table from providers data""" - - # Sort providers alphabetically by display name - sorted_providers = sorted( - providers_data.items(), - key=lambda x: x[1]['display_name'].lower() - ) - - # Generate header - header_cols = ["Provider"] + [col[0] for col in ENDPOINT_COLUMNS] - header = "| " + " | ".join(header_cols) + " |" - separator = "|" + "|".join(["-" * (len(col) + 2) for col in header_cols]) + "|" - - # Generate rows - rows = [] - for slug, data in sorted_providers: - display_name = data['display_name'] - url = data['url'] - - # Build row - row_parts = [f"[{display_name}]({url})"] - - for _, endpoint_key in ENDPOINT_COLUMNS: - supported = data['endpoints'].get(endpoint_key, False) - row_parts.append("โœ…" if supported else "") - - row = "| " + " | ".join(row_parts) + " |" - rows.append(row) - - # Combine all parts - table_lines = [ - "", - "", - "", - header, - separator - ] + rows + [ - "" - ] - - return "\n".join(table_lines) - - -def update_readme(table_markdown): - """Update README.md with new table""" - with open(README_PATH, 'r') as f: - content = f.read() - - print(f" Original README length: {len(content)} bytes") - - # Find the table section - # Look for the AUTO-GENERATED comment or the header, and replace until Read the Docs - pattern = r"(## Supported Providers.*?\n\n)(?:|.*?)(\n\n\[\*\*Read the Docs\*\*\])" - - # Test if pattern matches - match = re.search(pattern, content, flags=re.DOTALL) - if not match: - print("โŒ Pattern did not match in README.md") - return False - - print(f" Pattern matched, replacing table...") - - def replacer(match): - return match.group(1) + table_markdown + match.group(2) - - new_content = re.sub(pattern, replacer, content, flags=re.DOTALL) - - print(f" New README length: {len(new_content)} bytes") - - if new_content == content: - print(" โ„น๏ธ Table is already up-to-date, no changes needed") - return True # Not an error - table is already correct - - with open(README_PATH, 'w') as f: - f.write(new_content) - - print(" โœ“ README.md has been updated") - return True - - -def main(): - """Main function""" - print("Loading provider data from provider_endpoints_support.json...") - providers_data = load_providers_data() - print(f"โœ“ Loaded {len(providers_data)} providers") - - print("\nGenerating markdown table...") - table_markdown = generate_markdown_table(providers_data) - print(f"โœ“ Generated table with {len(providers_data)} rows") - - print("\nUpdating README.md...") - if update_readme(table_markdown): - print("โœ“ Successfully updated README.md") - print("\n๐Ÿ“ Please review the changes and commit both files:") - print(" - provider_endpoints_support.json") - print(" - README.md") - else: - print("โŒ Failed to update README.md") - return 1 - - return 0 - - -if __name__ == "__main__": - exit(main()) - diff --git a/test_litellm/proxy/pass_through_endpoints/test_passthrough_guardrails_field_targeting.py b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_guardrails_field_targeting.py similarity index 100% rename from test_litellm/proxy/pass_through_endpoints/test_passthrough_guardrails_field_targeting.py rename to tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_guardrails_field_targeting.py