Revert "Enhance proxy CLI with Rich formatting and improved user experience (#11420)"
This reverts commit 3b911ba1b2.
This commit is contained in:
Ishaan Jaff
parent
3be1dab1e1
commit
08d6f3e142
2
.github/workflows/test-linting.yml
vendored
2
.github/workflows/test-linting.yml
vendored
@ -23,7 +23,7 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install openai==1.81.0
|
||||
poetry install --with dev,proxy-dev --extras proxy
|
||||
poetry install --with dev
|
||||
pip install openai==1.81.0
|
||||
|
||||
|
||||
|
||||
@ -12,12 +12,6 @@ import click
|
||||
import httpx
|
||||
from dotenv import load_dotenv
|
||||
import urllib.parse
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.table import Table
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn
|
||||
from rich.align import Align
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import FastAPI
|
||||
else:
|
||||
@ -25,9 +19,6 @@ else:
|
||||
|
||||
sys.path.append(os.getcwd())
|
||||
|
||||
# Initialize Rich console
|
||||
console = Console()
|
||||
|
||||
config_filename = "litellm.secrets"
|
||||
|
||||
litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV"
|
||||
@ -59,64 +50,14 @@ def append_query_params(url, params) -> str:
|
||||
class ProxyInitializationHelpers:
|
||||
@staticmethod
|
||||
def _echo_litellm_version():
|
||||
"""Display LiteLLM version with rich formatting"""
|
||||
try:
|
||||
pkg_version = importlib.metadata.version("litellm") # type: ignore
|
||||
|
||||
# Create a beautiful version display
|
||||
version_panel = Panel(
|
||||
Align.center(f"[bold cyan]LiteLLM[/bold cyan]\n[green]Version: {pkg_version}[/green]"),
|
||||
title="[bold blue]LiteLLM Proxy[/bold blue]",
|
||||
border_style="cyan",
|
||||
padding=(1, 2)
|
||||
)
|
||||
console.print()
|
||||
console.print(version_panel)
|
||||
console.print()
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error getting version: {e}[/red]")
|
||||
pkg_version = importlib.metadata.version("litellm") # type: ignore
|
||||
click.echo(f"\nLiteLLM: Current Version = {pkg_version}\n")
|
||||
|
||||
@staticmethod
|
||||
def _run_health_check(host, port):
|
||||
"""Run health check with rich progress indicators"""
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
console=console,
|
||||
) as progress:
|
||||
task = progress.add_task("Running health check...", total=None)
|
||||
|
||||
try:
|
||||
response = httpx.get(url=f"http://{host}:{port}/health")
|
||||
progress.update(task, completed=True)
|
||||
|
||||
if response.status_code == 200:
|
||||
console.print("[green]✅ Health check passed![/green]")
|
||||
|
||||
# Create a formatted table for health check results
|
||||
health_data = response.json()
|
||||
|
||||
if isinstance(health_data, dict):
|
||||
table = Table(title="Health Check Results", show_header=True, header_style="bold magenta")
|
||||
table.add_column("Model", style="cyan")
|
||||
table.add_column("Status", style="green")
|
||||
table.add_column("Response Time", style="yellow")
|
||||
|
||||
for model, details in health_data.items():
|
||||
if isinstance(details, dict):
|
||||
status = "✅ Healthy" if details.get("status") == "healthy" else "❌ Unhealthy"
|
||||
response_time = details.get("response_time", "N/A")
|
||||
table.add_row(model, status, str(response_time))
|
||||
|
||||
console.print(table)
|
||||
else:
|
||||
console.print_json(data=health_data)
|
||||
else:
|
||||
console.print(f"[red]❌ Health check failed with status {response.status_code}[/red]")
|
||||
|
||||
except Exception as e:
|
||||
progress.update(task, completed=True)
|
||||
console.print(f"[red]❌ Health check failed: {e}[/red]")
|
||||
print("\nLiteLLM: Health Testing models in config") # noqa
|
||||
response = httpx.get(url=f"http://{host}:{port}/health")
|
||||
print(json.dumps(response.json(), indent=4)) # noqa
|
||||
|
||||
@staticmethod
|
||||
def _run_test_chat_completion(
|
||||
@ -125,17 +66,10 @@ class ProxyInitializationHelpers:
|
||||
model: str,
|
||||
test: Union[bool, str],
|
||||
):
|
||||
"""Run test chat completion with rich formatting and progress"""
|
||||
request_model = model or "gpt-3.5-turbo"
|
||||
|
||||
# Create test info panel
|
||||
test_panel = Panel(
|
||||
f"[cyan]Model:[/cyan] {request_model}\n[cyan]Endpoint:[/cyan] http://{host}:{port}",
|
||||
title="[bold yellow]Test Configuration[/bold yellow]",
|
||||
border_style="yellow"
|
||||
click.echo(
|
||||
f"\nLiteLLM: Making a test ChatCompletions request to your proxy. Model={request_model}"
|
||||
)
|
||||
console.print(test_panel)
|
||||
|
||||
import openai
|
||||
|
||||
api_base = f"http://{host}:{port}"
|
||||
@ -143,91 +77,41 @@ class ProxyInitializationHelpers:
|
||||
api_base = test
|
||||
else:
|
||||
raise ValueError("Invalid test value")
|
||||
|
||||
client = openai.OpenAI(api_key="My API Key", base_url=api_base)
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
console=console,
|
||||
) as progress:
|
||||
# Test 1: Regular completion
|
||||
task1 = progress.add_task("Testing chat completion...", total=None)
|
||||
try:
|
||||
response = client.chat.completions.create(
|
||||
model=request_model,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "this is a test request, write a short poem",
|
||||
}
|
||||
],
|
||||
max_tokens=256,
|
||||
)
|
||||
progress.update(task1, completed=True)
|
||||
console.print("[green]✅ Chat completion test passed![/green]")
|
||||
|
||||
# Display response in a nice format
|
||||
if response.choices and response.choices[0].message:
|
||||
response_panel = Panel(
|
||||
response.choices[0].message.content or "No content",
|
||||
title="[bold green]Response[/bold green]",
|
||||
border_style="green"
|
||||
)
|
||||
console.print(response_panel)
|
||||
|
||||
except Exception as e:
|
||||
progress.update(task1, completed=True)
|
||||
console.print(f"[red]❌ Chat completion test failed: {e}[/red]")
|
||||
response = client.chat.completions.create(
|
||||
model=request_model,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "this is a test request, write a short poem",
|
||||
}
|
||||
],
|
||||
max_tokens=256,
|
||||
)
|
||||
click.echo(f"\nLiteLLM: response from proxy {response}")
|
||||
|
||||
# Test 2: Streaming completion
|
||||
task2 = progress.add_task("Testing streaming completion...", total=None)
|
||||
try:
|
||||
stream_response = client.chat.completions.create(
|
||||
model=request_model,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "this is a test request, write a short poem",
|
||||
}
|
||||
],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
console.print("[cyan]Streaming response:[/cyan]")
|
||||
for chunk in stream_response:
|
||||
if chunk.choices and chunk.choices[0].delta.content:
|
||||
console.print(chunk.choices[0].delta.content, end="")
|
||||
|
||||
progress.update(task2, completed=True)
|
||||
console.print("\n[green]✅ Streaming completion test passed![/green]")
|
||||
|
||||
except Exception as e:
|
||||
progress.update(task2, completed=True)
|
||||
console.print(f"[red]❌ Streaming completion test failed: {e}[/red]")
|
||||
print( # noqa
|
||||
f"\n LiteLLM: Making a test ChatCompletions + streaming r equest to proxy. Model={request_model}"
|
||||
)
|
||||
|
||||
# Test 3: Legacy completion
|
||||
task3 = progress.add_task("Testing legacy completion...", total=None)
|
||||
try:
|
||||
completion_response = client.completions.create(
|
||||
model=request_model,
|
||||
prompt="this is a test request, write a short poem"
|
||||
)
|
||||
progress.update(task3, completed=True)
|
||||
console.print("[green]✅ Legacy completion test passed![/green]")
|
||||
|
||||
# Display response in a nice format
|
||||
if completion_response.choices and completion_response.choices[0].text:
|
||||
legacy_response_panel = Panel(
|
||||
completion_response.choices[0].text or "No content",
|
||||
title="[bold green]Legacy Response[/bold green]",
|
||||
border_style="green"
|
||||
)
|
||||
console.print(legacy_response_panel)
|
||||
|
||||
except Exception as e:
|
||||
progress.update(task3, completed=True)
|
||||
console.print(f"[red]❌ Legacy completion test failed: {e}[/red]")
|
||||
stream_response = client.chat.completions.create(
|
||||
model=request_model,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "this is a test request, write a short poem",
|
||||
}
|
||||
],
|
||||
stream=True,
|
||||
)
|
||||
for chunk in stream_response:
|
||||
click.echo(f"LiteLLM: streaming response from proxy {chunk}")
|
||||
print("\n making completion request to proxy") # noqa
|
||||
completion_response = client.completions.create(
|
||||
model=request_model, prompt="this is a test request, write a short poem"
|
||||
)
|
||||
print(completion_response) # noqa
|
||||
|
||||
@staticmethod
|
||||
def _get_default_unvicorn_init_args(
|
||||
@ -246,10 +130,10 @@ class ProxyInitializationHelpers:
|
||||
"port": port,
|
||||
}
|
||||
if log_config is not None:
|
||||
console.print(f"[cyan]Using log_config:[/cyan] {log_config}")
|
||||
print(f"Using log_config: {log_config}") # noqa
|
||||
uvicorn_args["log_config"] = log_config
|
||||
elif litellm.json_logs:
|
||||
console.print("[cyan]Using JSON logs. Setting log_config to None.[/cyan]")
|
||||
print("Using json logs. Setting log_config to None.") # noqa
|
||||
uvicorn_args["log_config"] = None
|
||||
return uvicorn_args
|
||||
|
||||
@ -269,28 +153,16 @@ class ProxyInitializationHelpers:
|
||||
from hypercorn.asyncio import serve
|
||||
from hypercorn.config import Config
|
||||
|
||||
# Display server start message with rich formatting
|
||||
server_panel = Panel(
|
||||
f"[green]Starting LiteLLM Proxy Server[/green]\n"
|
||||
f"[cyan]Server:[/cyan] Hypercorn\n"
|
||||
f"[cyan]Host:[/cyan] {host}\n"
|
||||
f"[cyan]Port:[/cyan] {port}",
|
||||
title="[bold blue]Server Configuration[/bold blue]",
|
||||
border_style="blue"
|
||||
)
|
||||
console.print(server_panel)
|
||||
|
||||
print( # noqa
|
||||
f"\033[1;32mLiteLLM Proxy: Starting server on {host}:{port} using Hypercorn\033[0m\n" # noqa
|
||||
) # noqa
|
||||
config = Config()
|
||||
config.bind = [f"{host}:{port}"]
|
||||
|
||||
if ssl_certfile_path is not None and ssl_keyfile_path is not None:
|
||||
ssl_panel = Panel(
|
||||
f"[cyan]Certificate:[/cyan] {ssl_certfile_path}\n"
|
||||
f"[cyan]Key File:[/cyan] {ssl_keyfile_path}",
|
||||
title="[bold green]SSL Configuration[/bold green]",
|
||||
border_style="green"
|
||||
print( # noqa
|
||||
f"\033[1;32mLiteLLM Proxy: Using SSL with certfile: {ssl_certfile_path} and keyfile: {ssl_keyfile_path}\033[0m\n" # noqa
|
||||
)
|
||||
console.print(ssl_panel)
|
||||
config.certfile = ssl_certfile_path
|
||||
config.keyfile = ssl_keyfile_path
|
||||
|
||||
@ -321,46 +193,38 @@ class ProxyInitializationHelpers:
|
||||
self.application = app # FastAPI app
|
||||
super().__init__()
|
||||
|
||||
# Create beautiful server info display
|
||||
server_info = Panel(
|
||||
f"[green]LiteLLM Proxy Server Starting[/green]\n"
|
||||
f"[cyan]Server:[/cyan] Gunicorn\n"
|
||||
f"[cyan]Host:[/cyan] {host}\n"
|
||||
f"[cyan]Port:[/cyan] {port}\n"
|
||||
f"[cyan]Workers:[/cyan] {num_workers}",
|
||||
title="[bold blue]Server Configuration[/bold blue]",
|
||||
border_style="blue"
|
||||
_endpoint_str = (
|
||||
f"curl --location 'http://0.0.0.0:{port}/chat/completions' \\"
|
||||
)
|
||||
console.print(server_info)
|
||||
|
||||
# Create testing instructions
|
||||
curl_command = f"""curl --location 'http://0.0.0.0:{port}/chat/completions' \\
|
||||
--header 'Content-Type: application/json' \\
|
||||
--data '{{
|
||||
"model": "gpt-3.5-turbo",
|
||||
"messages": [
|
||||
{{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}}
|
||||
]
|
||||
}}'"""
|
||||
|
||||
test_panel = Panel(
|
||||
f"[yellow]Quick Test:[/yellow] litellm --test\n\n"
|
||||
f"[yellow]cURL Test:[/yellow]\n{curl_command}",
|
||||
title="[bold yellow]Testing Instructions[/bold yellow]",
|
||||
border_style="yellow"
|
||||
curl_command = (
|
||||
_endpoint_str
|
||||
+ """
|
||||
--header 'Content-Type: application/json' \\
|
||||
--data ' {
|
||||
"model": "gpt-3.5-turbo",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}
|
||||
]
|
||||
}'
|
||||
\n
|
||||
"""
|
||||
)
|
||||
console.print(test_panel)
|
||||
|
||||
links_panel = Panel(
|
||||
f"[blue]Documentation:[/blue] https://docs.litellm.ai/docs/simple_proxy\n"
|
||||
f"[blue]Swagger UI:[/blue] http://0.0.0.0:{port}",
|
||||
title="[bold blue]Useful Links[/bold blue]",
|
||||
border_style="blue"
|
||||
print() # noqa
|
||||
print( # noqa
|
||||
'\033[1;34mLiteLLM: Test your local proxy with: "litellm --test" This runs an openai.ChatCompletion request to your proxy [In a new terminal tab]\033[0m\n'
|
||||
)
|
||||
console.print(links_panel)
|
||||
print( # noqa
|
||||
f"\033[1;34mLiteLLM: Curl Command Test for your local proxy\n {curl_command} \033[0m\n"
|
||||
)
|
||||
print( # noqa
|
||||
"\033[1;34mDocs: https://docs.litellm.ai/docs/simple_proxy\033[0m\n"
|
||||
) # noqa
|
||||
print( # noqa
|
||||
f"\033[1;34mSee all Router/Swagger docs on http://0.0.0.0:{port} \033[0m\n"
|
||||
) # noqa
|
||||
|
||||
def load_config(self):
|
||||
# note: This Loads the gunicorn config - has nothing to do with LiteLLM Proxy config
|
||||
@ -380,6 +244,9 @@ class ProxyInitializationHelpers:
|
||||
# gunicorn app function
|
||||
return self.application
|
||||
|
||||
print( # noqa
|
||||
f"\033[1;32mLiteLLM Proxy: Starting server on {host}:{port} with {num_workers} workers\033[0m\n" # noqa
|
||||
)
|
||||
gunicorn_options = {
|
||||
"bind": f"{host}:{port}",
|
||||
"workers": num_workers, # default is 1
|
||||
@ -391,13 +258,9 @@ class ProxyInitializationHelpers:
|
||||
}
|
||||
|
||||
if ssl_certfile_path is not None and ssl_keyfile_path is not None:
|
||||
ssl_panel = Panel(
|
||||
f"[cyan]Certificate:[/cyan] {ssl_certfile_path}\n"
|
||||
f"[cyan]Key File:[/cyan] {ssl_keyfile_path}",
|
||||
title="[bold green]SSL Configuration[/bold green]",
|
||||
border_style="green"
|
||||
print( # noqa
|
||||
f"\033[1;32mLiteLLM Proxy: Using SSL with certfile: {ssl_certfile_path} and keyfile: {ssl_keyfile_path}\033[0m\n" # noqa
|
||||
)
|
||||
console.print(ssl_panel)
|
||||
gunicorn_options["certfile"] = ssl_certfile_path
|
||||
gunicorn_options["keyfile"] = ssl_keyfile_path
|
||||
|
||||
@ -410,15 +273,12 @@ class ProxyInitializationHelpers:
|
||||
|
||||
with open(os.devnull, "w") as devnull:
|
||||
subprocess.Popen(command, stdout=devnull, stderr=devnull)
|
||||
console.print("[green]✅ Ollama serve started successfully[/green]")
|
||||
except Exception as e:
|
||||
console.print(Panel(
|
||||
f"[red]Failed to start Ollama serve[/red]\n"
|
||||
f"[yellow]Error:[/yellow] {e}\n"
|
||||
f"[yellow]Please ensure Ollama is installed and run:[/yellow] ollama serve",
|
||||
title="[bold red]Ollama Warning[/bold red]",
|
||||
border_style="red"
|
||||
))
|
||||
print( # noqa
|
||||
f"""
|
||||
LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve`
|
||||
"""
|
||||
) # noqa
|
||||
|
||||
@staticmethod
|
||||
def _is_port_in_use(port):
|
||||
@ -434,272 +294,176 @@ class ProxyInitializationHelpers:
|
||||
return None # Let uvicorn choose the default loop on Windows
|
||||
return "uvloop"
|
||||
|
||||
@staticmethod
|
||||
def _display_startup_banner(host: str, port: int, config_path: Optional[str] = None):
|
||||
"""Display a beautiful startup banner"""
|
||||
# Create configuration info
|
||||
config_info = f"[cyan]Host:[/cyan] {host}\n[cyan]Port:[/cyan] {port}"
|
||||
if config_path:
|
||||
config_info += f"\n[cyan]Config:[/cyan] {config_path}"
|
||||
|
||||
# Create startup panel
|
||||
startup_panel = Panel(
|
||||
Align.center(f"[bold cyan]🚄 LiteLLM Proxy[/bold cyan]\n\n{config_info}"),
|
||||
title="[bold green]Starting Server[/bold green]",
|
||||
border_style="green",
|
||||
padding=(1, 2)
|
||||
)
|
||||
|
||||
console.print()
|
||||
console.print(startup_panel)
|
||||
console.print()
|
||||
|
||||
|
||||
def create_help_panel():
|
||||
"""Create a beautiful help panel with grouped options"""
|
||||
help_content = """
|
||||
[bold cyan]Server Options:[/bold cyan]
|
||||
--host, --port Server binding configuration
|
||||
--num_workers Number of worker processes
|
||||
--config, -c Configuration file path
|
||||
|
||||
[bold cyan]Model Options:[/bold cyan]
|
||||
--model, -m Model name
|
||||
--alias Model alias
|
||||
--api_base API base URL
|
||||
|
||||
[bold cyan]Logging & Debug:[/bold cyan]
|
||||
--debug Enable debug mode
|
||||
--detailed_debug Enable detailed debugging
|
||||
--log_config Logging configuration file
|
||||
|
||||
[bold cyan]Testing:[/bold cyan]
|
||||
--test Run test completion
|
||||
--health Run health check
|
||||
--version, -v Show version
|
||||
|
||||
[bold cyan]Security:[/bold cyan]
|
||||
--ssl_certfile_path SSL certificate file
|
||||
--ssl_keyfile_path SSL key file
|
||||
"""
|
||||
|
||||
help_panel = Panel(
|
||||
help_content.strip(),
|
||||
title="[bold blue]LiteLLM Proxy CLI Help[/bold blue]",
|
||||
border_style="blue",
|
||||
padding=(1, 2)
|
||||
)
|
||||
return help_panel
|
||||
|
||||
|
||||
# Enhanced click command with rich help
|
||||
class RichCommand(click.Command):
|
||||
def format_help(self, ctx, formatter):
|
||||
console.print(create_help_panel())
|
||||
|
||||
|
||||
@click.command(cls=RichCommand)
|
||||
@click.command()
|
||||
@click.option(
|
||||
"--host",
|
||||
default="0.0.0.0",
|
||||
help="🌐 Host for the server to listen on",
|
||||
envvar="HOST",
|
||||
show_default=True
|
||||
)
|
||||
@click.option(
|
||||
"--port",
|
||||
default=4000,
|
||||
help="🔌 Port to bind the server to",
|
||||
envvar="PORT",
|
||||
show_default=True
|
||||
"--host", default="0.0.0.0", help="Host for the server to listen on.", envvar="HOST"
|
||||
)
|
||||
@click.option("--port", default=4000, help="Port to bind the server to.", envvar="PORT")
|
||||
@click.option(
|
||||
"--num_workers",
|
||||
default=1,
|
||||
help="👥 Number of uvicorn/gunicorn workers",
|
||||
help="Number of uvicorn / gunicorn workers to spin up. By default, 1 uvicorn is used.",
|
||||
envvar="NUM_WORKERS",
|
||||
show_default=True
|
||||
)
|
||||
@click.option("--api_base", default=None, help="🔗 API base URL")
|
||||
@click.option("--api_base", default=None, help="API base URL.")
|
||||
@click.option(
|
||||
"--api_version",
|
||||
default="2024-07-01-preview",
|
||||
help="📅 Azure API version",
|
||||
show_default=True
|
||||
help="For azure - pass in the api version.",
|
||||
)
|
||||
@click.option(
|
||||
"--model", "-m",
|
||||
default=None,
|
||||
help="🤖 Model name to use"
|
||||
"--model", "-m", default=None, help="The model name to pass to litellm expects"
|
||||
)
|
||||
@click.option(
|
||||
"--alias",
|
||||
default=None,
|
||||
help='📝 Model alias (e.g., "codellama" for long model names)',
|
||||
help='The alias for the model - use this to give a litellm model name (e.g. "huggingface/codellama/CodeLlama-7b-Instruct-hf") a more user-friendly name ("codellama")',
|
||||
)
|
||||
@click.option(
|
||||
"--add_key",
|
||||
default=None,
|
||||
help="🔑 Add API key"
|
||||
)
|
||||
@click.option("--headers", default=None, help="📋 Headers for API calls")
|
||||
@click.option(
|
||||
"--save",
|
||||
is_flag=True,
|
||||
help="💾 Save model-specific configuration"
|
||||
"--add_key", default=None, help="The model name to pass to litellm expects"
|
||||
)
|
||||
@click.option("--headers", default=None, help="headers for the API call")
|
||||
@click.option("--save", is_flag=True, type=bool, help="Save the model-specific config")
|
||||
@click.option(
|
||||
"--debug",
|
||||
default=False,
|
||||
is_flag=True,
|
||||
help="🐛 Enable debug mode",
|
||||
type=bool,
|
||||
help="To debug the input",
|
||||
envvar="DEBUG",
|
||||
)
|
||||
@click.option(
|
||||
"--detailed_debug",
|
||||
default=False,
|
||||
is_flag=True,
|
||||
help="🔍 Enable detailed debug logs",
|
||||
type=bool,
|
||||
help="To view detailed debug logs",
|
||||
envvar="DETAILED_DEBUG",
|
||||
)
|
||||
@click.option(
|
||||
"--use_queue",
|
||||
default=False,
|
||||
is_flag=True,
|
||||
help="⚡ Use celery workers for async endpoints",
|
||||
type=bool,
|
||||
help="To use celery workers for async endpoints",
|
||||
)
|
||||
@click.option(
|
||||
"--temperature",
|
||||
default=None,
|
||||
type=float,
|
||||
help="🌡️ Model temperature"
|
||||
"--temperature", default=None, type=float, help="Set temperature for the model"
|
||||
)
|
||||
@click.option(
|
||||
"--max_tokens",
|
||||
default=None,
|
||||
type=int,
|
||||
help="📏 Maximum tokens"
|
||||
"--max_tokens", default=None, type=int, help="Set max tokens for the model"
|
||||
)
|
||||
@click.option(
|
||||
"--request_timeout",
|
||||
default=None,
|
||||
type=int,
|
||||
help="⏱️ Request timeout (seconds)",
|
||||
)
|
||||
@click.option(
|
||||
"--drop_params",
|
||||
is_flag=True,
|
||||
help="🗑️ Drop unmapped parameters"
|
||||
help="Set timeout in seconds for completion calls",
|
||||
)
|
||||
@click.option("--drop_params", is_flag=True, help="Drop any unmapped params")
|
||||
@click.option(
|
||||
"--add_function_to_prompt",
|
||||
is_flag=True,
|
||||
help="🔧 Add unsupported functions to prompt",
|
||||
help="If function passed but unsupported, pass it as prompt",
|
||||
)
|
||||
@click.option(
|
||||
"--config",
|
||||
"-c",
|
||||
default=None,
|
||||
help="⚙️ Configuration file path (e.g., config.yaml)",
|
||||
help="Path to the proxy configuration file (e.g. config.yaml). Usage `litellm --config config.yaml`",
|
||||
)
|
||||
@click.option(
|
||||
"--max_budget",
|
||||
default=None,
|
||||
type=float,
|
||||
help="💰 Maximum budget for API calls",
|
||||
help="Set max budget for API calls - works for hosted models like OpenAI, TogetherAI, Anthropic, etc.`",
|
||||
)
|
||||
@click.option(
|
||||
"--telemetry",
|
||||
default=True,
|
||||
type=bool,
|
||||
help="📊 Enable telemetry (helps improve LiteLLM)",
|
||||
show_default=True
|
||||
help="Helps us know if people are using this feature. Turn this off by doing `--telemetry False`",
|
||||
)
|
||||
@click.option(
|
||||
"--log_config",
|
||||
default=None,
|
||||
type=str,
|
||||
help="📝 Logging configuration file path",
|
||||
help="Path to the logging configuration file",
|
||||
)
|
||||
@click.option(
|
||||
"--version",
|
||||
"-v",
|
||||
default=False,
|
||||
is_flag=True,
|
||||
help="📋 Show LiteLLM version",
|
||||
type=bool,
|
||||
help="Print LiteLLM version",
|
||||
)
|
||||
@click.option(
|
||||
"--health",
|
||||
flag_value=True,
|
||||
help="🏥 Run health check on all models",
|
||||
help="Make a chat/completions request to all llms in config.yaml",
|
||||
)
|
||||
@click.option(
|
||||
"--test",
|
||||
flag_value=True,
|
||||
help="🧪 Run test chat completion",
|
||||
help="proxy chat completions url to make a test request to",
|
||||
)
|
||||
@click.option(
|
||||
"--test_async",
|
||||
default=False,
|
||||
is_flag=True,
|
||||
help="⚡ Test async endpoints",
|
||||
help="Calls async endpoints /queue/requests and /queue/response",
|
||||
)
|
||||
@click.option(
|
||||
"--iam_token_db_auth",
|
||||
default=False,
|
||||
is_flag=True,
|
||||
help="🔐 Use IAM token for database authentication",
|
||||
help="Connects to RDS DB with IAM token",
|
||||
)
|
||||
@click.option(
|
||||
"--num_requests",
|
||||
default=10,
|
||||
type=int,
|
||||
help="🔢 Number of requests for async testing",
|
||||
show_default=True
|
||||
help="Number of requests to hit async endpoint with",
|
||||
)
|
||||
@click.option(
|
||||
"--run_gunicorn",
|
||||
default=False,
|
||||
is_flag=True,
|
||||
help="🦄 Use Gunicorn instead of Uvicorn",
|
||||
help="Starts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)",
|
||||
)
|
||||
@click.option(
|
||||
"--run_hypercorn",
|
||||
default=False,
|
||||
is_flag=True,
|
||||
help="🚄 Use Hypercorn (HTTP/2 support)",
|
||||
help="Starts proxy via hypercorn, instead of uvicorn (supports HTTP/2)",
|
||||
)
|
||||
@click.option(
|
||||
"--ssl_keyfile_path",
|
||||
default=None,
|
||||
type=str,
|
||||
help="🔐 SSL private key file path",
|
||||
help="Path to the SSL keyfile. Use this when you want to provide SSL certificate when starting proxy",
|
||||
envvar="SSL_KEYFILE_PATH",
|
||||
)
|
||||
@click.option(
|
||||
"--ssl_certfile_path",
|
||||
default=None,
|
||||
type=str,
|
||||
help="📜 SSL certificate file path",
|
||||
help="Path to the SSL certfile. Use this when you want to provide SSL certificate when starting proxy",
|
||||
envvar="SSL_CERTFILE_PATH",
|
||||
)
|
||||
@click.option(
|
||||
"--use_prisma_migrate",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="🗃️ Use Prisma migrate for schema updates",
|
||||
)
|
||||
@click.option(
|
||||
"--local",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="🏠 Local debugging mode"
|
||||
help="Use prisma migrate instead of prisma db push for database schema updates",
|
||||
)
|
||||
@click.option("--local", is_flag=True, default=False, help="for local debugging")
|
||||
@click.option(
|
||||
"--skip_server_startup",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="⏭️ Skip server startup (migrations only)",
|
||||
help="Skip starting the server after setup (useful for migrations only)",
|
||||
)
|
||||
def run_server( # noqa: PLR0915
|
||||
host,
|
||||
@ -738,14 +502,7 @@ def run_server( # noqa: PLR0915
|
||||
use_prisma_migrate,
|
||||
skip_server_startup,
|
||||
):
|
||||
"""
|
||||
🚄 LiteLLM Proxy Server - A unified interface for 100+ LLMs
|
||||
|
||||
Start a proxy server that provides OpenAI-compatible endpoints for various LLM providers.
|
||||
"""
|
||||
args = locals()
|
||||
|
||||
# Handle imports
|
||||
if local:
|
||||
from proxy_server import (
|
||||
KeyManagementSettings,
|
||||
@ -764,12 +521,6 @@ def run_server( # noqa: PLR0915
|
||||
except ImportError as e:
|
||||
if "litellm[proxy]" in str(e):
|
||||
# user is missing a proxy dependency, ask them to pip install litellm[proxy]
|
||||
console.print(Panel(
|
||||
"[red]Missing proxy dependencies![/red]\n\n"
|
||||
"Please install with: [cyan]pip install 'litellm[proxy]'[/cyan]",
|
||||
title="[bold red]Installation Error[/bold red]",
|
||||
border_style="red"
|
||||
))
|
||||
raise e
|
||||
else:
|
||||
# this is just a local/relative import error, user git cloned litellm
|
||||
@ -779,34 +530,20 @@ def run_server( # noqa: PLR0915
|
||||
app,
|
||||
save_worker_config,
|
||||
)
|
||||
|
||||
# Handle version display
|
||||
if version is True:
|
||||
ProxyInitializationHelpers._echo_litellm_version()
|
||||
return
|
||||
|
||||
# Handle Ollama setup
|
||||
if model and "ollama" in model and api_base is None:
|
||||
ProxyInitializationHelpers._run_ollama_serve()
|
||||
|
||||
# Handle health check
|
||||
if health is True:
|
||||
ProxyInitializationHelpers._run_health_check(host, port)
|
||||
return
|
||||
|
||||
# Handle test completion
|
||||
if test is True:
|
||||
ProxyInitializationHelpers._run_test_chat_completion(host, port, model, test)
|
||||
return
|
||||
|
||||
# Main server startup flow
|
||||
else:
|
||||
# Display startup banner
|
||||
ProxyInitializationHelpers._display_startup_banner(host, port, config)
|
||||
|
||||
if headers:
|
||||
headers = json.loads(headers)
|
||||
|
||||
save_worker_config(
|
||||
model=model,
|
||||
alias=alias,
|
||||
@ -826,16 +563,9 @@ def run_server( # noqa: PLR0915
|
||||
config=config,
|
||||
use_queue=use_queue,
|
||||
)
|
||||
|
||||
try:
|
||||
import uvicorn
|
||||
except Exception:
|
||||
console.print(Panel(
|
||||
"[red]Missing server dependencies![/red]\n\n"
|
||||
"Please install with: [cyan]pip install 'litellm[proxy]'[/cyan]",
|
||||
title="[bold red]Import Error[/bold red]",
|
||||
border_style="red"
|
||||
))
|
||||
raise ImportError(
|
||||
"uvicorn, gunicorn needs to be imported. Run - `pip install 'litellm[proxy]'`"
|
||||
)
|
||||
@ -843,8 +573,8 @@ def run_server( # noqa: PLR0915
|
||||
db_connection_pool_limit = 100
|
||||
db_connection_timeout = 60
|
||||
general_settings = {}
|
||||
|
||||
### GET DB TOKEN FOR IAM AUTH ###
|
||||
|
||||
if iam_token_db_auth:
|
||||
from litellm.proxy.auth.rds_iam_token import generate_iam_auth_token
|
||||
|
||||
@ -858,6 +588,7 @@ def run_server( # noqa: PLR0915
|
||||
db_host=db_host, db_port=db_port, db_user=db_user
|
||||
)
|
||||
|
||||
# print(f"token: {token}")
|
||||
_db_url = f"postgresql://{db_user}:{token}@{db_host}:{db_port}/{db_name}"
|
||||
if db_schema:
|
||||
_db_url += f"?schema={db_schema}"
|
||||
@ -866,6 +597,7 @@ def run_server( # noqa: PLR0915
|
||||
os.environ["IAM_TOKEN_DB_AUTH"] = "True"
|
||||
|
||||
### DECRYPT ENV VAR ###
|
||||
|
||||
from litellm.secret_managers.aws_secret_manager import decrypt_env_var
|
||||
|
||||
if (
|
||||
@ -905,8 +637,8 @@ def run_server( # noqa: PLR0915
|
||||
import litellm
|
||||
|
||||
litellm.json_logs = True
|
||||
|
||||
litellm._turn_on_json()
|
||||
|
||||
### GENERAL SETTINGS ###
|
||||
general_settings = _config.get("general_settings", {})
|
||||
if general_settings is None:
|
||||
@ -917,7 +649,6 @@ def run_server( # noqa: PLR0915
|
||||
"key_management_system", None
|
||||
)
|
||||
proxy_config.initialize_secret_manager(key_management_system)
|
||||
|
||||
key_management_settings = general_settings.get(
|
||||
"key_management_settings", None
|
||||
)
|
||||
@ -927,7 +658,6 @@ def run_server( # noqa: PLR0915
|
||||
litellm._key_management_settings = KeyManagementSettings(
|
||||
**key_management_settings
|
||||
)
|
||||
|
||||
database_url = general_settings.get("database_url", None)
|
||||
if database_url is None and os.getenv("DATABASE_URL") is None:
|
||||
# Check if all required variables are provided
|
||||
@ -951,7 +681,6 @@ def run_server( # noqa: PLR0915
|
||||
database_url = f"postgresql://{database_username_enc}:{database_password_enc}@{database_host}/{database_name_enc}"
|
||||
|
||||
os.environ["DATABASE_URL"] = database_url
|
||||
|
||||
db_connection_pool_limit = general_settings.get(
|
||||
"database_connection_pool_limit",
|
||||
LiteLLMDatabaseConnectionPool.database_connection_pool_limit.value,
|
||||
@ -978,14 +707,6 @@ def run_server( # noqa: PLR0915
|
||||
os.getenv("DATABASE_URL", None) is not None
|
||||
or os.getenv("DIRECT_URL", None) is not None
|
||||
):
|
||||
# Display a nice message before database setup
|
||||
db_setup_panel = Panel(
|
||||
"[cyan]Setting up database connection and schema...[/cyan]",
|
||||
title="[bold blue]Database Setup[/bold blue]",
|
||||
border_style="blue"
|
||||
)
|
||||
console.print(db_setup_panel)
|
||||
|
||||
try:
|
||||
from litellm.secret_managers.main import get_secret
|
||||
|
||||
@ -1030,44 +751,35 @@ def run_server( # noqa: PLR0915
|
||||
else:
|
||||
PrismaManager.setup_database(use_migrate=use_prisma_migrate)
|
||||
else:
|
||||
console.print("[yellow]⚠️ Unable to connect to DB. DATABASE_URL found but Prisma not available.[/yellow]")
|
||||
|
||||
# Check port availability
|
||||
print( # noqa
|
||||
f"Unable to connect to DB. DATABASE_URL found in environment, but prisma package not found." # noqa
|
||||
)
|
||||
if port == 4000 and ProxyInitializationHelpers._is_port_in_use(port):
|
||||
old_port = port
|
||||
port = random.randint(1024, 49152)
|
||||
console.print(f"[yellow]⚠️ Port {old_port} is in use. Using port {port} instead.[/yellow]")
|
||||
|
||||
import litellm
|
||||
|
||||
if detailed_debug is True:
|
||||
litellm._turn_on_debug()
|
||||
console.print("[cyan]🔍 Detailed debugging enabled[/cyan]")
|
||||
|
||||
# DO NOT DELETE - enables global variables to work across files
|
||||
from litellm.proxy.proxy_server import app # noqa
|
||||
|
||||
# Skip server startup if requested (after all setup is done)
|
||||
if skip_server_startup:
|
||||
console.print("[yellow]⏭️ Setup complete. Skipping server startup as requested.[/yellow]")
|
||||
print("LiteLLM: Setup complete. Skipping server startup as requested.") # noqa
|
||||
return
|
||||
|
||||
# Final server startup
|
||||
uvicorn_args = ProxyInitializationHelpers._get_default_unvicorn_init_args(
|
||||
host=host,
|
||||
port=port,
|
||||
log_config=log_config,
|
||||
)
|
||||
|
||||
if run_gunicorn is False and run_hypercorn is False:
|
||||
if ssl_certfile_path is not None and ssl_keyfile_path is not None:
|
||||
ssl_panel = Panel(
|
||||
f"[cyan]Certificate:[/cyan] {ssl_certfile_path}\n"
|
||||
f"[cyan]Key File:[/cyan] {ssl_keyfile_path}",
|
||||
title="[bold green]SSL Configuration[/bold green]",
|
||||
border_style="green"
|
||||
print( # noqa
|
||||
f"\033[1;32mLiteLLM Proxy: Using SSL with certfile: {ssl_certfile_path} and keyfile: {ssl_keyfile_path}\033[0m\n" # noqa
|
||||
)
|
||||
console.print(ssl_panel)
|
||||
uvicorn_args["ssl_keyfile"] = ssl_keyfile_path
|
||||
uvicorn_args["ssl_certfile"] = ssl_certfile_path
|
||||
|
||||
@ -1075,18 +787,6 @@ def run_server( # noqa: PLR0915
|
||||
if loop_type:
|
||||
uvicorn_args["loop"] = loop_type
|
||||
|
||||
# Final startup message
|
||||
startup_msg = Panel(
|
||||
f"[green]🚄 Starting LiteLLM Proxy Server[/green]\n"
|
||||
f"[cyan]Server:[/cyan] Uvicorn\n"
|
||||
f"[cyan]Host:[/cyan] {host}\n"
|
||||
f"[cyan]Port:[/cyan] {port}\n"
|
||||
f"[cyan]Workers:[/cyan] {num_workers}",
|
||||
title="[bold blue]Server Starting[/bold blue]",
|
||||
border_style="blue"
|
||||
)
|
||||
console.print(startup_msg)
|
||||
|
||||
uvicorn.run(
|
||||
**uvicorn_args,
|
||||
workers=num_workers,
|
||||
@ -1111,10 +811,4 @@ def run_server( # noqa: PLR0915
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
run_server()
|
||||
except KeyboardInterrupt:
|
||||
console.print("\n[yellow]👋 Server stopped by user[/yellow]")
|
||||
except Exception as e:
|
||||
console.print(f"\n[red]❌ Error: {e}[/red]")
|
||||
raise
|
||||
run_server()
|
||||
|
||||
@ -48,6 +48,15 @@ else:
|
||||
OpenTelemetry = Any
|
||||
|
||||
|
||||
def showwarning(message, category, filename, lineno, file=None, line=None):
|
||||
traceback_info = f"{filename}:{lineno}: {category.__name__}: {message}\n"
|
||||
if file is not None:
|
||||
file.write(traceback_info)
|
||||
|
||||
|
||||
warnings.showwarning = showwarning
|
||||
warnings.filterwarnings("default", category=UserWarning)
|
||||
|
||||
# Your client code here
|
||||
|
||||
|
||||
@ -67,6 +76,45 @@ try:
|
||||
except ImportError as e:
|
||||
raise ImportError(f"Missing dependency {e}. Run `pip install 'litellm[proxy]'`")
|
||||
|
||||
list_of_messages = [
|
||||
"'The thing I wish you improved is...'",
|
||||
"'A feature I really want is...'",
|
||||
"'The worst thing about this product is...'",
|
||||
"'This product would be better if...'",
|
||||
"'I don't like how this works...'",
|
||||
"'It would help me if you could add...'",
|
||||
"'This feature doesn't meet my needs because...'",
|
||||
"'I get frustrated when the product...'",
|
||||
]
|
||||
|
||||
|
||||
def generate_feedback_box():
|
||||
box_width = 60
|
||||
|
||||
# Select a random message
|
||||
message = random.choice(list_of_messages)
|
||||
|
||||
print() # noqa
|
||||
print("\033[1;37m" + "#" + "-" * box_width + "#\033[0m") # noqa
|
||||
print("\033[1;37m" + "#" + " " * box_width + "#\033[0m") # noqa
|
||||
print("\033[1;37m" + "# {:^59} #\033[0m".format(message)) # noqa
|
||||
print( # noqa
|
||||
"\033[1;37m"
|
||||
+ "# {:^59} #\033[0m".format("https://github.com/BerriAI/litellm/issues/new")
|
||||
) # noqa
|
||||
print("\033[1;37m" + "#" + " " * box_width + "#\033[0m") # noqa
|
||||
print("\033[1;37m" + "#" + "-" * box_width + "#\033[0m") # noqa
|
||||
print() # noqa
|
||||
print(" Thank you for using LiteLLM! - Krrish & Ishaan") # noqa
|
||||
print() # noqa
|
||||
print() # noqa
|
||||
print() # noqa
|
||||
print( # noqa
|
||||
"\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m"
|
||||
) # noqa
|
||||
print() # noqa
|
||||
print() # noqa
|
||||
|
||||
|
||||
from collections import defaultdict
|
||||
from contextlib import asynccontextmanager
|
||||
@ -262,8 +310,6 @@ from litellm.proxy.utils import (
|
||||
_get_redoc_url,
|
||||
_is_projected_spend_over_limit,
|
||||
_is_valid_team_configs,
|
||||
display_model_initialization,
|
||||
generate_feedback_box,
|
||||
get_custom_url,
|
||||
get_error_message_str,
|
||||
get_server_root_path,
|
||||
@ -1945,12 +1991,15 @@ class ProxyConfig:
|
||||
model_list = config.get("model_list", None)
|
||||
if model_list:
|
||||
router_params["model_list"] = model_list
|
||||
|
||||
# Display beautiful model initialization using utils function
|
||||
display_model_initialization(model_list, get_secret)
|
||||
|
||||
# Check for Ollama models that need local server
|
||||
print( # noqa
|
||||
"\033[32mLiteLLM: Proxy initialized with Config, Set models:\033[0m"
|
||||
) # noqa
|
||||
for model in model_list:
|
||||
### LOAD FROM os.environ/ ###
|
||||
for k, v in model["litellm_params"].items():
|
||||
if isinstance(v, str) and v.startswith("os.environ/"):
|
||||
model["litellm_params"][k] = get_secret(v)
|
||||
print(f"\033[32m {model.get('model_name', '')}\033[0m") # noqa
|
||||
litellm_model_name = model["litellm_params"]["model"]
|
||||
litellm_model_api_base = model["litellm_params"].get("api_base", None)
|
||||
if "ollama" in litellm_model_name and litellm_model_api_base is None:
|
||||
|
||||
@ -3,7 +3,6 @@ import copy
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import smtplib
|
||||
import threading
|
||||
import time
|
||||
@ -96,191 +95,6 @@ else:
|
||||
Span = Any
|
||||
|
||||
|
||||
### CLI DISPLAY FUNCTIONS ###
|
||||
|
||||
# List of random feedback messages for the feedback box
|
||||
list_of_messages = [
|
||||
"'The thing I wish you improved is...'",
|
||||
"'A feature I really want is...'",
|
||||
"'The worst thing about this product is...'",
|
||||
"'This product would be better if...'",
|
||||
"'I don't like how this works...'",
|
||||
"'It would help me if you could add...'",
|
||||
"'This feature doesn't meet my needs because...'",
|
||||
"'I get frustrated when the product...'",
|
||||
]
|
||||
|
||||
|
||||
def showwarning(message, category, filename, lineno, file=None, line=None):
|
||||
"""
|
||||
Custom warning handler for CLI display.
|
||||
|
||||
Args:
|
||||
message: Warning message
|
||||
category: Warning category
|
||||
filename: File where warning occurred
|
||||
lineno: Line number where warning occurred
|
||||
file: Optional file object to write to
|
||||
line: Optional line content
|
||||
"""
|
||||
traceback_info = f"{filename}:{lineno}: {category.__name__}: {message}\n"
|
||||
if file is not None:
|
||||
file.write(traceback_info)
|
||||
|
||||
|
||||
def display_model_initialization(model_list: List[dict], get_secret_function):
|
||||
"""
|
||||
Display beautiful model initialization information using Rich console.
|
||||
|
||||
This function displays a formatted table of configured models with provider
|
||||
information using rich formatting if available, with a fallback to basic
|
||||
logging for environments without rich.
|
||||
|
||||
Args:
|
||||
model_list: List of model configuration dictionaries
|
||||
get_secret_function: Function to retrieve secrets from environment variables
|
||||
"""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.table import Table
|
||||
from rich.align import Align
|
||||
from litellm import get_llm_provider
|
||||
|
||||
console = Console()
|
||||
|
||||
# Create a beautiful table for models
|
||||
models_table = Table(title="🤖 Configured Models", show_header=True, header_style="bold cyan")
|
||||
models_table.add_column("Model Name", style="green", min_width=25)
|
||||
models_table.add_column("Provider", style="blue", min_width=15)
|
||||
|
||||
for model in model_list:
|
||||
### LOAD FROM os.environ/ ###
|
||||
for k, v in model["litellm_params"].items():
|
||||
if isinstance(v, str) and v.startswith("os.environ/"):
|
||||
model["litellm_params"][k] = get_secret_function(v)
|
||||
|
||||
model_name = model.get('model_name', 'Unknown')
|
||||
litellm_model = model["litellm_params"].get("model", "Unknown")
|
||||
|
||||
# Extract provider using get_llm_provider utility with try/except
|
||||
provider = "OpenAI" # default
|
||||
try:
|
||||
_, custom_llm_provider, _, _ = get_llm_provider(model=litellm_model)
|
||||
if custom_llm_provider:
|
||||
provider = custom_llm_provider.replace("_", " ").title()
|
||||
except Exception:
|
||||
# Fallback to default provider if get_llm_provider fails
|
||||
provider = "OpenAI"
|
||||
|
||||
models_table.add_row(model_name, provider)
|
||||
|
||||
# Create initialization panel
|
||||
init_panel = Panel(
|
||||
Align.center(models_table),
|
||||
title="[bold green]LiteLLM Proxy Initialized[/bold green]",
|
||||
border_style="green",
|
||||
padding=(1, 2)
|
||||
)
|
||||
|
||||
console.print()
|
||||
console.print(init_panel)
|
||||
console.print()
|
||||
|
||||
except ImportError:
|
||||
# Fallback to original formatting if Rich is not available
|
||||
from litellm import get_llm_provider
|
||||
|
||||
verbose_proxy_logger.info("LiteLLM: Proxy initialized with Config, Set models:")
|
||||
for model in model_list:
|
||||
### LOAD FROM os.environ/ ###
|
||||
for k, v in model["litellm_params"].items():
|
||||
if isinstance(v, str) and v.startswith("os.environ/"):
|
||||
model["litellm_params"][k] = get_secret_function(v)
|
||||
verbose_proxy_logger.info(" %s", model.get('model_name', ''))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def generate_feedback_box():
|
||||
"""
|
||||
Generate and display a beautiful feedback box with random message prompts.
|
||||
|
||||
This function displays a formatted feedback request box using rich formatting
|
||||
if available, with a fallback to ASCII art for environments without rich.
|
||||
"""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.text import Text
|
||||
from rich.align import Align
|
||||
|
||||
console = Console()
|
||||
|
||||
# Select a random message
|
||||
message = random.choice(list_of_messages)
|
||||
|
||||
# Create feedback panel with beautiful formatting
|
||||
feedback_content = f"[yellow]{message}[/yellow]\n\n[cyan]https://github.com/BerriAI/litellm/issues/new[/cyan]"
|
||||
|
||||
feedback_panel = Panel(
|
||||
Align.center(feedback_content),
|
||||
title="[bold blue]Feature Request[/bold blue]",
|
||||
border_style="blue",
|
||||
padding=(1, 2)
|
||||
)
|
||||
|
||||
# Create thank you message
|
||||
thank_you_text = Text("Thank you for using LiteLLM! 🚄", style="bold green")
|
||||
thank_you_subtitle = Text("- Krrish & Ishaan", style="italic cyan")
|
||||
|
||||
# Create help panel
|
||||
help_panel = Panel(
|
||||
"[red]Give Feedback / Get Help:[/red] [cyan]https://github.com/BerriAI/litellm/issues/new[/cyan]",
|
||||
title="[bold red]Need Help?[/bold red]",
|
||||
border_style="red",
|
||||
padding=(0, 2)
|
||||
)
|
||||
|
||||
console.print()
|
||||
console.print(feedback_panel)
|
||||
console.print()
|
||||
console.print(Align.center(thank_you_text))
|
||||
console.print(Align.center(thank_you_subtitle))
|
||||
console.print()
|
||||
console.print(help_panel)
|
||||
console.print()
|
||||
|
||||
except ImportError:
|
||||
# Fallback to original implementation if rich is not available
|
||||
box_width = 60
|
||||
|
||||
# Select a random message
|
||||
message = random.choice(list_of_messages)
|
||||
|
||||
print() # noqa
|
||||
print("\033[1;37m" + "#" + "-" * box_width + "#\033[0m") # noqa
|
||||
print("\033[1;37m" + "#" + " " * box_width + "#\033[0m") # noqa
|
||||
print("\033[1;37m" + "# {:^59} #\033[0m".format(message)) # noqa
|
||||
print( # noqa
|
||||
"\033[1;37m"
|
||||
+ "# {:^59} #\033[0m".format("https://github.com/BerriAI/litellm/issues/new")
|
||||
) # noqa
|
||||
print("\033[1;37m" + "#" + " " * box_width + "#\033[0m") # noqa
|
||||
print("\033[1;37m" + "#" + "-" * box_width + "#\033[0m") # noqa
|
||||
print() # noqa
|
||||
print(" Thank you for using LiteLLM! - Krrish & Ishaan") # noqa
|
||||
print() # noqa
|
||||
print() # noqa
|
||||
print() # noqa
|
||||
print( # noqa
|
||||
"\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m"
|
||||
) # noqa
|
||||
print() # noqa
|
||||
print() # noqa
|
||||
|
||||
|
||||
def print_verbose(print_statement):
|
||||
"""
|
||||
Prints the given `print_statement` to the console if `litellm.set_verbose` is True.
|
||||
|
||||
@ -13,8 +13,8 @@ from litellm.proxy.proxy_cli import ProxyInitializationHelpers
|
||||
|
||||
class TestProxyInitializationHelpers:
|
||||
@patch("importlib.metadata.version")
|
||||
@patch("litellm.proxy.proxy_cli.console.print")
|
||||
def test_echo_litellm_version(self, mock_console_print, mock_version):
|
||||
@patch("click.echo")
|
||||
def test_echo_litellm_version(self, mock_echo, mock_version):
|
||||
# Setup
|
||||
mock_version.return_value = "1.0.0"
|
||||
|
||||
@ -23,17 +23,17 @@ class TestProxyInitializationHelpers:
|
||||
|
||||
# Assert
|
||||
mock_version.assert_called_once_with("litellm")
|
||||
# Should call console.print multiple times (for empty lines and panel)
|
||||
assert mock_console_print.call_count >= 3
|
||||
mock_echo.assert_called_once_with("\nLiteLLM: Current Version = 1.0.0\n")
|
||||
|
||||
@patch("httpx.get")
|
||||
@patch("litellm.proxy.proxy_cli.console.print")
|
||||
def test_run_health_check(self, mock_console_print, mock_get):
|
||||
@patch("builtins.print")
|
||||
@patch("json.dumps")
|
||||
def test_run_health_check(self, mock_dumps, mock_print, mock_get):
|
||||
# Setup
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {"model1": {"status": "healthy", "response_time": "0.5s"}}
|
||||
mock_response.json.return_value = {"status": "healthy"}
|
||||
mock_get.return_value = mock_response
|
||||
mock_dumps.return_value = '{"status": "healthy"}'
|
||||
|
||||
# Execute
|
||||
ProxyInitializationHelpers._run_health_check("localhost", 8000)
|
||||
@ -41,8 +41,7 @@ class TestProxyInitializationHelpers:
|
||||
# Assert
|
||||
mock_get.assert_called_once_with(url="http://localhost:8000/health")
|
||||
mock_response.json.assert_called_once()
|
||||
# Should call console.print multiple times (progress, success message, table)
|
||||
assert mock_console_print.call_count >= 2
|
||||
mock_dumps.assert_called_once_with({"status": "healthy"}, indent=4)
|
||||
|
||||
@patch("openai.OpenAI")
|
||||
@patch("click.echo")
|
||||
@ -198,8 +197,8 @@ class TestProxyInitializationHelpers:
|
||||
assert "pool_timeout=60" in modified_url
|
||||
|
||||
@patch("uvicorn.run")
|
||||
@patch("litellm.proxy.proxy_cli.console.print")
|
||||
def test_skip_server_startup(self, mock_console_print, mock_uvicorn_run):
|
||||
@patch("builtins.print")
|
||||
def test_skip_server_startup(self, mock_print, mock_uvicorn_run):
|
||||
"""Test that the skip_server_startup flag prevents server startup when True"""
|
||||
from click.testing import CliRunner
|
||||
|
||||
@ -235,11 +234,12 @@ class TestProxyInitializationHelpers:
|
||||
|
||||
assert result.exit_code == 0
|
||||
mock_uvicorn_run.assert_not_called()
|
||||
# Check that console.print was called (for skip message)
|
||||
assert mock_console_print.call_count >= 1
|
||||
mock_print.assert_any_call(
|
||||
"LiteLLM: Setup complete. Skipping server startup as requested."
|
||||
)
|
||||
|
||||
mock_uvicorn_run.reset_mock()
|
||||
mock_console_print.reset_mock()
|
||||
mock_print.reset_mock()
|
||||
|
||||
result = runner.invoke(run_server, ["--local"])
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user