[Feat] New Provider - Add RunwayML Provider for video generations (#16505)
* add RUNWAYML * init folders * add RunwayMLVideoConfig * add RUNWAYML_DEFAULT_API_VERSION * add RunwayMLVideoConfig * fix getting status * add async_transform_video_content_response * add runwayml transform_video_content_response * fix config.yaml * add runwayml docs * add runwayml to videos * docs runwayml video gen * add new models to model cost map * TestRunwayMLVideoTransformation * fix linting errors
This commit is contained in:
Ishaan Jaff
GitHub
parent
663f2d7e7f
commit
50b5cf5215
266
docs/my-website/docs/providers/runwayml/videos.md
Normal file
266
docs/my-website/docs/providers/runwayml/videos.md
Normal file
@ -0,0 +1,266 @@
|
||||
# RunwayML - Video Generation
|
||||
|
||||
LiteLLM supports RunwayML's Gen-4 video generation API, allowing you to generate videos from text prompts and images.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```python showLineNumbers title="Basic Video Generation"
|
||||
from litellm import video_generation
|
||||
import os
|
||||
|
||||
os.environ["RUNWAYML_API_KEY"] = "your-api-key"
|
||||
|
||||
# Generate video from text and image
|
||||
response = video_generation(
|
||||
model="runwayml/gen4_turbo",
|
||||
prompt="A high quality demo video of litellm ai gateway",
|
||||
input_reference="https://media.licdn.com/dms/image/v2/D4D0BAQFqOrIAJEgtLw/company-logo_200_200/company-logo_200_200/0/1714076049190/berri_ai_logo?e=2147483647&v=beta&t=7tG_KRZZ4MPGc7Iin79PcFcrpvf5Hu6rBM4ptHGU1DY",
|
||||
seconds=5,
|
||||
size="1280x720"
|
||||
)
|
||||
|
||||
print(f"Video ID: {response.id}")
|
||||
print(f"Status: {response.status}")
|
||||
```
|
||||
|
||||
## Authentication
|
||||
|
||||
Set your RunwayML API key:
|
||||
|
||||
```python showLineNumbers title="Set API Key"
|
||||
import os
|
||||
|
||||
os.environ["RUNWAYML_API_KEY"] = "your-api-key"
|
||||
```
|
||||
|
||||
## Supported Parameters
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|------|----------|-------------|
|
||||
| `model` | string | Yes | Model to use (e.g., `runwayml/gen4_turbo`) |
|
||||
| `prompt` | string | Yes | Text description for the video |
|
||||
| `input_reference` | string/file | Yes | URL or file path to reference image |
|
||||
| `seconds` | int | No | Video duration (5 or 10 seconds) |
|
||||
| `size` | string | No | Video dimensions (`1280x720` or `720x1280`). Can also use `ratio` format (`1280:720`) |
|
||||
|
||||
## Complete Workflow
|
||||
|
||||
```python showLineNumbers title="Complete Video Generation Workflow"
|
||||
from litellm import video_generation, video_status, video_content
|
||||
import os
|
||||
import time
|
||||
|
||||
os.environ["RUNWAYML_API_KEY"] = "your-api-key"
|
||||
|
||||
# 1. Generate video
|
||||
response = video_generation(
|
||||
model="runwayml/gen4_turbo",
|
||||
prompt="A high quality demo video of litellm ai gateway",
|
||||
input_reference="https://media.licdn.com/dms/image/v2/D4D0BAQFqOrIAJEgtLw/company-logo_200_200/company-logo_200_200/0/1714076049190/berri_ai_logo?e=2147483647&v=beta&t=7tG_KRZZ4MPGc7Iin79PcFcrpvf5Hu6rBM4ptHGU1DY",
|
||||
seconds=5,
|
||||
size="1280x720"
|
||||
)
|
||||
|
||||
video_id = response.id
|
||||
print(f"Video generation started: {video_id}")
|
||||
|
||||
# 2. Check status until completed
|
||||
while True:
|
||||
status_response = video_status(video_id=video_id)
|
||||
print(f"Status: {status_response.status}")
|
||||
|
||||
if status_response.status == "completed":
|
||||
print("Video generation completed!")
|
||||
break
|
||||
elif status_response.status == "failed":
|
||||
print("Video generation failed")
|
||||
break
|
||||
|
||||
time.sleep(10) # Wait 10 seconds before checking again
|
||||
|
||||
# 3. Download video content
|
||||
video_bytes = video_content(video_id=video_id)
|
||||
|
||||
# 4. Save to file
|
||||
with open("generated_video.mp4", "wb") as f:
|
||||
f.write(video_bytes)
|
||||
|
||||
print("Video saved successfully!")
|
||||
```
|
||||
|
||||
## Async Usage
|
||||
|
||||
```python showLineNumbers title="Async Video Generation"
|
||||
from litellm import avideo_generation, avideo_status, avideo_content
|
||||
import os
|
||||
import asyncio
|
||||
|
||||
os.environ["RUNWAYML_API_KEY"] = "your-api-key"
|
||||
|
||||
async def generate_video():
|
||||
# Generate video
|
||||
response = await avideo_generation(
|
||||
model="runwayml/gen4_turbo",
|
||||
prompt="A serene lake with mountains in the background",
|
||||
input_reference="https://example.com/lake.jpg",
|
||||
seconds=5,
|
||||
size="1280x720"
|
||||
)
|
||||
|
||||
video_id = response.id
|
||||
print(f"Video generation started: {video_id}")
|
||||
|
||||
# Poll for completion
|
||||
while True:
|
||||
status_response = await avideo_status(video_id=video_id)
|
||||
print(f"Status: {status_response.status}")
|
||||
|
||||
if status_response.status == "completed":
|
||||
break
|
||||
elif status_response.status == "failed":
|
||||
print("Video generation failed")
|
||||
return
|
||||
|
||||
await asyncio.sleep(10)
|
||||
|
||||
# Download video
|
||||
video_bytes = await avideo_content(video_id=video_id)
|
||||
|
||||
# Save to file
|
||||
with open("generated_video.mp4", "wb") as f:
|
||||
f.write(video_bytes)
|
||||
|
||||
print("Video saved successfully!")
|
||||
|
||||
asyncio.run(generate_video())
|
||||
```
|
||||
|
||||
## LiteLLM Proxy Usage
|
||||
|
||||
Add RunwayML to your proxy configuration:
|
||||
|
||||
```yaml showLineNumbers title="config.yaml"
|
||||
model_list:
|
||||
- model_name: gen4-turbo
|
||||
litellm_params:
|
||||
model: runwayml/gen4_turbo
|
||||
api_key: os.environ/RUNWAYML_API_KEY
|
||||
```
|
||||
|
||||
Start the proxy:
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
Generate videos through the proxy:
|
||||
|
||||
```bash showLineNumbers title="Proxy Request"
|
||||
curl --location 'http://localhost:4000/v1/videos' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header 'x-litellm-api-key: sk-1234' \
|
||||
--data '{
|
||||
"model": "runwayml/gen4_turbo",
|
||||
"prompt": "A high quality demo video of litellm ai gateway",
|
||||
"input_reference": "https://media.licdn.com/dms/image/v2/D4D0BAQFqOrIAJEgtLw/company-logo_200_200/company-logo_200_200/0/1714076049190/berri_ai_logo?e=2147483647&v=beta&t=7tG_KRZZ4MPGc7Iin79PcFcrpvf5Hu6rBM4ptHGU1DY",
|
||||
"ratio": "1280:720"
|
||||
}'
|
||||
```
|
||||
|
||||
Check video status:
|
||||
|
||||
```bash showLineNumbers title="Check Status"
|
||||
curl --location 'http://localhost:4000/v1/videos/{video_id}' \
|
||||
--header 'x-litellm-api-key: sk-1234'
|
||||
```
|
||||
|
||||
Download video content:
|
||||
|
||||
```bash showLineNumbers title="Download Video"
|
||||
curl --location 'http://localhost:4000/v1/videos/{video_id}/content' \
|
||||
--header 'x-litellm-api-key: sk-1234' \
|
||||
--output video.mp4
|
||||
```
|
||||
|
||||
## Supported Models
|
||||
|
||||
| Model | Description | Duration | Aspect Ratios |
|
||||
|-------|-------------|----------|---------------|
|
||||
| `runwayml/gen4_turbo` | Fast video generation | 5-10s | 1280x720, 720x1280 |
|
||||
|
||||
## Error Handling
|
||||
|
||||
```python showLineNumbers title="Error Handling"
|
||||
from litellm import video_generation, video_status
|
||||
import time
|
||||
|
||||
try:
|
||||
response = video_generation(
|
||||
model="runwayml/gen4_turbo",
|
||||
prompt="A scenic mountain view",
|
||||
input_reference="https://example.com/mountain.jpg",
|
||||
seconds=5
|
||||
)
|
||||
|
||||
# Poll for completion
|
||||
max_attempts = 60 # 10 minutes max
|
||||
attempts = 0
|
||||
|
||||
while attempts < max_attempts:
|
||||
status_response = video_status(video_id=response.id)
|
||||
|
||||
if status_response.status == "completed":
|
||||
print("Video generation completed!")
|
||||
break
|
||||
elif status_response.status == "failed":
|
||||
error = status_response.error or {}
|
||||
print(f"Video generation failed: {error.get('message', 'Unknown error')}")
|
||||
break
|
||||
|
||||
time.sleep(10)
|
||||
attempts += 1
|
||||
|
||||
if attempts >= max_attempts:
|
||||
print("Video generation timed out")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {str(e)}")
|
||||
```
|
||||
|
||||
## Cost Tracking
|
||||
|
||||
LiteLLM automatically tracks RunwayML video generation costs:
|
||||
|
||||
```python showLineNumbers title="Cost Tracking"
|
||||
from litellm import video_generation, completion_cost
|
||||
|
||||
response = video_generation(
|
||||
model="runwayml/gen4_turbo",
|
||||
prompt="A high quality demo video of litellm ai gateway",
|
||||
input_reference="https://media.licdn.com/dms/image/v2/D4D0BAQFqOrIAJEgtLw/company-logo_200_200/company-logo_200_200/0/1714076049190/berri_ai_logo?e=2147483647&v=beta&t=7tG_KRZZ4MPGc7Iin79PcFcrpvf5Hu6rBM4ptHGU1DY",
|
||||
seconds=5,
|
||||
size="1280x720"
|
||||
)
|
||||
|
||||
# Calculate cost
|
||||
cost = completion_cost(completion_response=response)
|
||||
print(f"Video generation cost: ${cost}")
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
For complete API details, see the [OpenAI Video Generation API specification](https://platform.openai.com/docs/guides/video-generation) which LiteLLM follows.
|
||||
|
||||
## Supported Features
|
||||
|
||||
| Feature | Supported |
|
||||
|---------|-----------|
|
||||
| Video Generation | ✅ |
|
||||
| Image-to-Video | ✅ |
|
||||
| Status Checking | ✅ |
|
||||
| Content Download | ✅ |
|
||||
| Cost Tracking | ✅ |
|
||||
| Logging | ✅ |
|
||||
| Fallbacks | ✅ |
|
||||
| Load Balancing | ✅ |
|
||||
|
||||
@ -9,7 +9,7 @@ Fallbacks | ✅ (Between supported models) |
|
||||
| Guardrails Support | ✅ Content moderation and safety checks |
|
||||
| Proxy Server Support | ✅ Full proxy integration with virtual keys |
|
||||
| Spend Management | ✅ Budget tracking and rate limiting |
|
||||
| Supported Providers | `openai`, `azure`, `gemini`, `vertex_ai` |
|
||||
| Supported Providers | `openai`, `azure`, `gemini`, `vertex_ai`, `runwayml` |
|
||||
|
||||
:::tip
|
||||
|
||||
@ -605,3 +605,4 @@ The response follows OpenAI's video generation format with the following structu
|
||||
| Azure | [Usage](providers/azure/videos) |
|
||||
| Gemini | [Usage](providers/gemini/videos) |
|
||||
| Vertex AI | [Usage](providers/vertex_ai/videos) |
|
||||
| RunwayML | [Usage](providers/runwayml/videos) |
|
||||
|
||||
@ -576,6 +576,13 @@ const sidebars = {
|
||||
"providers/nlp_cloud",
|
||||
"providers/recraft",
|
||||
"providers/replicate",
|
||||
{
|
||||
type: "category",
|
||||
label: "RunwayML",
|
||||
items: [
|
||||
"providers/runwayml/videos",
|
||||
]
|
||||
},
|
||||
"providers/togetherai",
|
||||
"providers/v0",
|
||||
"providers/vercel_ai_gateway",
|
||||
|
||||
@ -85,6 +85,7 @@ MAX_TOKEN_TRIMMING_ATTEMPTS = int(
|
||||
os.getenv("MAX_TOKEN_TRIMMING_ATTEMPTS", 10)
|
||||
) # Maximum number of attempts to trim the message
|
||||
|
||||
RUNWAYML_DEFAULT_API_VERSION = str(os.getenv("RUNWAYML_DEFAULT_API_VERSION", "2024-11-06"))
|
||||
|
||||
########## Networking constants ##############################################################
|
||||
_DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client for 1 hour
|
||||
|
||||
@ -5,9 +5,9 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
|
||||
import httpx
|
||||
from httpx._types import RequestFiles
|
||||
|
||||
from litellm.types.videos.main import VideoCreateOptionalRequestParams
|
||||
from litellm.types.responses.main import *
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.videos.main import VideoCreateOptionalRequestParams
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
@ -134,6 +134,31 @@ class BaseVideoConfig(ABC):
|
||||
) -> bytes:
|
||||
pass
|
||||
|
||||
async def async_transform_video_content_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> bytes:
|
||||
"""
|
||||
Async transform video content download response to bytes.
|
||||
Optional method - providers can override if they need async transformations
|
||||
(e.g., RunwayML for downloading video from CloudFront URL).
|
||||
|
||||
Default implementation falls back to sync transform_video_content_response.
|
||||
|
||||
Args:
|
||||
raw_response: Raw HTTP response
|
||||
logging_obj: Logging object
|
||||
|
||||
Returns:
|
||||
Video content as bytes
|
||||
"""
|
||||
# Default implementation: call sync version
|
||||
return self.transform_video_content_response(
|
||||
raw_response=raw_response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def transform_video_remix_request(
|
||||
self,
|
||||
|
||||
@ -4414,7 +4414,7 @@ class BaseLLMHTTPHandler:
|
||||
)
|
||||
|
||||
# Transform the response using the provider config
|
||||
return video_content_provider_config.transform_video_content_response(
|
||||
return await video_content_provider_config.async_transform_video_content_response(
|
||||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
2
litellm/llms/runway/__init__.py
Normal file
2
litellm/llms/runway/__init__.py
Normal file
@ -0,0 +1,2 @@
|
||||
# RunwayML integration for LiteLLM
|
||||
|
||||
2
litellm/llms/runway/videos/__init__.py
Normal file
2
litellm/llms/runway/videos/__init__.py
Normal file
@ -0,0 +1,2 @@
|
||||
# RunwayML video generation
|
||||
|
||||
578
litellm/llms/runway/videos/transformation.py
Normal file
578
litellm/llms/runway/videos/transformation.py
Normal file
@ -0,0 +1,578 @@
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import httpx
|
||||
from httpx._types import RequestFiles
|
||||
|
||||
import litellm
|
||||
from litellm.constants import RUNWAYML_DEFAULT_API_VERSION
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
_get_httpx_client,
|
||||
get_async_httpx_client,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject
|
||||
from litellm.types.videos.utils import (
|
||||
encode_video_id_with_provider,
|
||||
extract_original_video_id,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
||||
from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException
|
||||
from ...base_llm.videos.transformation import BaseVideoConfig as _BaseVideoConfig
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
BaseVideoConfig = _BaseVideoConfig
|
||||
BaseLLMException = _BaseLLMException
|
||||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
BaseVideoConfig = Any
|
||||
BaseLLMException = Any
|
||||
|
||||
|
||||
class RunwayMLVideoConfig(BaseVideoConfig):
|
||||
"""
|
||||
Configuration class for RunwayML video generation.
|
||||
|
||||
RunwayML uses a task-based API where:
|
||||
1. POST /v1/image_to_video creates a task
|
||||
2. The task returns immediately with a task ID
|
||||
3. Client must poll or wait for task completion
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
Get the list of supported OpenAI parameters for video generation.
|
||||
Maps OpenAI params to RunwayML equivalents:
|
||||
- prompt -> promptText
|
||||
- input_reference -> promptImage
|
||||
- size -> ratio (e.g., "1280x720" -> "1280:720")
|
||||
- seconds -> duration
|
||||
"""
|
||||
return [
|
||||
"model",
|
||||
"prompt",
|
||||
"input_reference",
|
||||
"seconds",
|
||||
"size",
|
||||
"user",
|
||||
"extra_headers",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
video_create_optional_params: VideoCreateOptionalRequestParams,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> Dict:
|
||||
"""
|
||||
Map OpenAI parameters to RunwayML format.
|
||||
|
||||
Mappings:
|
||||
- prompt -> promptText
|
||||
- input_reference -> promptImage
|
||||
- size -> ratio (convert "WIDTHxHEIGHT" to "WIDTH:HEIGHT")
|
||||
- seconds -> duration (convert to integer)
|
||||
"""
|
||||
mapped_params: Dict[str, Any] = {}
|
||||
|
||||
# Handle input_reference parameter - map to promptImage
|
||||
if "input_reference" in video_create_optional_params:
|
||||
input_reference = video_create_optional_params["input_reference"]
|
||||
# RunwayML supports URLs and data URIs directly
|
||||
mapped_params["promptImage"] = input_reference
|
||||
|
||||
# Handle size parameter - convert "1280x720" to "1280:720"
|
||||
if "size" in video_create_optional_params:
|
||||
size = video_create_optional_params["size"]
|
||||
if isinstance(size, str) and "x" in size:
|
||||
mapped_params["ratio"] = size.replace("x", ":")
|
||||
|
||||
# Handle seconds parameter - convert to integer
|
||||
if "seconds" in video_create_optional_params:
|
||||
seconds = video_create_optional_params["seconds"]
|
||||
if seconds is not None:
|
||||
try:
|
||||
mapped_params["duration"] = int(float(seconds)) if isinstance(seconds, str) else int(seconds)
|
||||
except (ValueError, TypeError):
|
||||
# If conversion fails, use default duration
|
||||
pass
|
||||
|
||||
# Pass through other parameters that aren't OpenAI-specific
|
||||
supported_openai_params = self.get_supported_openai_params(model)
|
||||
for key, value in video_create_optional_params.items():
|
||||
if key not in supported_openai_params:
|
||||
mapped_params[key] = value
|
||||
|
||||
return mapped_params
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
api_key: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Validate environment and set up authentication headers.
|
||||
RunwayML uses Bearer token authentication via RUNWAYML_API_SECRET.
|
||||
"""
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm.api_key
|
||||
or get_secret_str("RUNWAYML_API_SECRET")
|
||||
or get_secret_str("RUNWAYML_API_KEY")
|
||||
)
|
||||
|
||||
if api_key is None:
|
||||
raise ValueError(
|
||||
"RunwayML API key is required. Set RUNWAYML_API_SECRET environment variable "
|
||||
"or pass api_key parameter."
|
||||
)
|
||||
|
||||
headers.update({
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"X-Runway-Version": RUNWAYML_DEFAULT_API_VERSION,
|
||||
"Content-Type": "application/json",
|
||||
})
|
||||
return headers
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
model: str,
|
||||
api_base: Optional[str],
|
||||
litellm_params: dict,
|
||||
) -> str:
|
||||
"""
|
||||
Get the base URL for RunwayML API.
|
||||
The specific endpoint path will be added in the transform methods.
|
||||
"""
|
||||
if api_base is None:
|
||||
api_base = "https://api.dev.runwayml.com/v1"
|
||||
|
||||
return api_base.rstrip('/')
|
||||
|
||||
def transform_video_create_request(
|
||||
self,
|
||||
model: str,
|
||||
prompt: str,
|
||||
api_base: str,
|
||||
video_create_optional_request_params: Dict,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[Dict, RequestFiles, str]:
|
||||
"""
|
||||
Transform the video creation request for RunwayML API.
|
||||
|
||||
RunwayML expects:
|
||||
{
|
||||
"model": "gen4_turbo",
|
||||
"promptImage": "https://... or data:image/...",
|
||||
"promptText": "description",
|
||||
"ratio": "1280:720",
|
||||
"duration": 5
|
||||
}
|
||||
"""
|
||||
# Build the request data
|
||||
request_data: Dict[str, Any] = {
|
||||
"model": model,
|
||||
"promptText": prompt,
|
||||
}
|
||||
|
||||
# Add mapped parameters
|
||||
request_data.update(video_create_optional_request_params)
|
||||
|
||||
# RunwayML uses JSON body, no files multipart
|
||||
files_list: List[Tuple[str, Any]] = []
|
||||
|
||||
# Append the specific endpoint for video generation
|
||||
full_api_base = f"{api_base}/image_to_video"
|
||||
|
||||
return request_data, files_list, full_api_base
|
||||
|
||||
def transform_video_create_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
request_data: Optional[Dict] = None,
|
||||
) -> VideoObject:
|
||||
"""
|
||||
Transform the RunwayML video creation response.
|
||||
|
||||
RunwayML returns a task object that looks like:
|
||||
{
|
||||
"id": "task_123...",
|
||||
"status": "PENDING" | "RUNNING" | "SUCCEEDED" | "FAILED",
|
||||
"output": ["https://...video.mp4"] (when succeeded)
|
||||
}
|
||||
|
||||
We map this to OpenAI VideoObject format.
|
||||
"""
|
||||
response_data = raw_response.json()
|
||||
|
||||
# Map RunwayML task response to VideoObject format
|
||||
video_data: Dict[str, Any] = {
|
||||
"id": response_data.get("id", ""),
|
||||
"object": "video",
|
||||
"status": self._map_runway_status(response_data.get("status", "pending")),
|
||||
"created_at": self._parse_runway_timestamp(response_data.get("createdAt")),
|
||||
}
|
||||
|
||||
# Add optional fields if present
|
||||
if "output" in response_data and response_data["output"]:
|
||||
# RunwayML returns output as array of URLs when task succeeds
|
||||
video_data["output_url"] = response_data["output"][0] if isinstance(response_data["output"], list) else response_data["output"]
|
||||
|
||||
if "completedAt" in response_data:
|
||||
video_data["completed_at"] = self._parse_runway_timestamp(response_data.get("completedAt"))
|
||||
|
||||
if "failureCode" in response_data or "failure" in response_data:
|
||||
video_data["error"] = {
|
||||
"code": response_data.get("failureCode", "unknown"),
|
||||
"message": response_data.get("failure", "Video generation failed")
|
||||
}
|
||||
|
||||
# Add model and size info if available from request
|
||||
if request_data:
|
||||
if "model" in request_data:
|
||||
video_data["model"] = request_data["model"]
|
||||
if "ratio" in request_data:
|
||||
# Convert ratio back to size format
|
||||
ratio = request_data["ratio"]
|
||||
if isinstance(ratio, str) and ":" in ratio:
|
||||
video_data["size"] = ratio.replace(":", "x")
|
||||
if "duration" in request_data:
|
||||
video_data["seconds"] = str(request_data["duration"])
|
||||
|
||||
video_obj = VideoObject(**video_data) # type: ignore[arg-type]
|
||||
|
||||
if custom_llm_provider and video_obj.id:
|
||||
video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model)
|
||||
|
||||
# Add usage data for cost tracking
|
||||
usage_data = {}
|
||||
if video_obj and hasattr(video_obj, 'seconds') and video_obj.seconds:
|
||||
try:
|
||||
usage_data["duration_seconds"] = float(video_obj.seconds)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
video_obj.usage = usage_data
|
||||
|
||||
return video_obj
|
||||
|
||||
def _map_runway_status(self, runway_status: str) -> str:
|
||||
"""
|
||||
Map RunwayML status to OpenAI status format.
|
||||
|
||||
RunwayML statuses: PENDING, RUNNING, SUCCEEDED, FAILED, CANCELLED
|
||||
OpenAI statuses: queued, in_progress, completed, failed
|
||||
"""
|
||||
status_map = {
|
||||
"PENDING": "queued",
|
||||
"RUNNING": "in_progress",
|
||||
"SUCCEEDED": "completed",
|
||||
"FAILED": "failed",
|
||||
"CANCELLED": "failed",
|
||||
"THROTTLED": "queued",
|
||||
}
|
||||
return status_map.get(runway_status.upper(), "queued")
|
||||
|
||||
def _parse_runway_timestamp(self, timestamp_str: Optional[str]) -> int:
|
||||
"""
|
||||
Convert RunwayML ISO 8601 timestamp to Unix timestamp.
|
||||
|
||||
RunwayML returns timestamps like: "2025-11-11T21:48:50.448Z"
|
||||
We need to convert to Unix timestamp (seconds since epoch).
|
||||
"""
|
||||
if not timestamp_str:
|
||||
return 0
|
||||
|
||||
try:
|
||||
# Parse ISO 8601 timestamp
|
||||
dt = datetime.fromisoformat(timestamp_str.replace('Z', '+00:00'))
|
||||
# Convert to Unix timestamp
|
||||
return int(dt.timestamp())
|
||||
except (ValueError, AttributeError):
|
||||
return 0
|
||||
|
||||
def transform_video_content_request(
|
||||
self,
|
||||
video_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform the video content request for RunwayML API.
|
||||
|
||||
RunwayML doesn't have a separate content download endpoint.
|
||||
The video URL is returned in the task output field.
|
||||
We'll retrieve the task and extract the video URL.
|
||||
"""
|
||||
original_video_id = extract_original_video_id(video_id)
|
||||
|
||||
# Get task status to retrieve video URL
|
||||
url = f"{api_base}/tasks/{original_video_id}"
|
||||
|
||||
params: Dict[str, Any] = {}
|
||||
|
||||
return url, params
|
||||
|
||||
def _extract_video_url_from_response(self, response_data: Dict[str, Any]) -> str:
|
||||
"""
|
||||
Helper method to extract video URL from RunwayML response.
|
||||
Shared between sync and async transforms.
|
||||
"""
|
||||
# Extract video URL from the output field
|
||||
video_url = None
|
||||
if "output" in response_data and response_data["output"]:
|
||||
output = response_data["output"]
|
||||
video_url = output[0] if isinstance(output, list) else output
|
||||
|
||||
if not video_url:
|
||||
# Check if the video generation failed or is still processing
|
||||
status = response_data.get("status", "UNKNOWN")
|
||||
if status in ["PENDING", "RUNNING", "THROTTLED"]:
|
||||
raise ValueError(f"Video is still processing (status: {status}). Please wait and try again.")
|
||||
elif status == "FAILED":
|
||||
failure_reason = response_data.get("failure", "Unknown error")
|
||||
raise ValueError(f"Video generation failed: {failure_reason}")
|
||||
else:
|
||||
raise ValueError("Video URL not found in response. Video may not be ready yet.")
|
||||
|
||||
return video_url
|
||||
|
||||
def transform_video_content_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> bytes:
|
||||
"""
|
||||
Transform the RunwayML video content download response (synchronous).
|
||||
|
||||
RunwayML's task endpoint returns JSON with a video URL in the output field.
|
||||
We need to extract the URL and download the video.
|
||||
|
||||
Example response:
|
||||
{
|
||||
"id":"63fd0f13-f29d-4e58-99d3-1cb9efa14a5b",
|
||||
"createdAt":"2025-11-11T21:48:50.448Z",
|
||||
"status":"SUCCEEDED",
|
||||
"output":["https://dnznrvs05pmza.cloudfront.net/.../video.mp4?_jwt=..."]
|
||||
}
|
||||
"""
|
||||
response_data = raw_response.json()
|
||||
video_url = self._extract_video_url_from_response(response_data)
|
||||
|
||||
# Download the video from the CloudFront URL synchronously
|
||||
httpx_client: HTTPHandler = _get_httpx_client()
|
||||
video_response = httpx_client.get(video_url)
|
||||
video_response.raise_for_status()
|
||||
|
||||
return video_response.content
|
||||
|
||||
async def async_transform_video_content_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> bytes:
|
||||
"""
|
||||
Transform the RunwayML video content download response (asynchronous).
|
||||
|
||||
RunwayML's task endpoint returns JSON with a video URL in the output field.
|
||||
We need to extract the URL and download the video asynchronously.
|
||||
|
||||
Example response:
|
||||
{
|
||||
"id":"63fd0f13-f29d-4e58-99d3-1cb9efa14a5b",
|
||||
"createdAt":"2025-11-11T21:48:50.448Z",
|
||||
"status":"SUCCEEDED",
|
||||
"output":["https://dnznrvs05pmza.cloudfront.net/.../video.mp4?_jwt=..."]
|
||||
}
|
||||
"""
|
||||
response_data = raw_response.json()
|
||||
video_url = self._extract_video_url_from_response(response_data)
|
||||
|
||||
# Download the video from the CloudFront URL asynchronously
|
||||
async_httpx_client: AsyncHTTPHandler = get_async_httpx_client(
|
||||
llm_provider=litellm.LlmProviders.RUNWAYML,
|
||||
)
|
||||
video_response = await async_httpx_client.get(video_url)
|
||||
video_response.raise_for_status()
|
||||
|
||||
return video_response.content
|
||||
|
||||
def transform_video_remix_request(
|
||||
self,
|
||||
video_id: str,
|
||||
prompt: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
extra_body: Optional[Dict[str, Any]] = None,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform the video remix request for RunwayML API.
|
||||
|
||||
RunwayML doesn't have a direct remix endpoint in their current API.
|
||||
This would need to be implemented when/if they add this feature.
|
||||
"""
|
||||
raise NotImplementedError("Video remix is not yet supported by RunwayML API")
|
||||
|
||||
def transform_video_remix_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
) -> VideoObject:
|
||||
"""Transform the RunwayML video remix response."""
|
||||
raise NotImplementedError("Video remix is not yet supported by RunwayML API")
|
||||
|
||||
def transform_video_list_request(
|
||||
self,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
after: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
order: Optional[str] = None,
|
||||
extra_query: Optional[Dict[str, Any]] = None,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform the video list request for RunwayML API.
|
||||
|
||||
RunwayML doesn't expose a list endpoint in their public API yet.
|
||||
"""
|
||||
raise NotImplementedError("Video listing is not yet supported by RunwayML API")
|
||||
|
||||
def transform_video_list_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
) -> Dict[str, str]:
|
||||
"""Transform the RunwayML video list response."""
|
||||
raise NotImplementedError("Video listing is not yet supported by RunwayML API")
|
||||
|
||||
def transform_video_delete_request(
|
||||
self,
|
||||
video_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform the video delete request for RunwayML API.
|
||||
|
||||
RunwayML uses task cancellation.
|
||||
"""
|
||||
original_video_id = extract_original_video_id(video_id)
|
||||
|
||||
# Construct the URL for task cancellation
|
||||
url = f"{api_base}/tasks/{original_video_id}/cancel"
|
||||
|
||||
data: Dict[str, Any] = {}
|
||||
|
||||
return url, data
|
||||
|
||||
def transform_video_delete_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> VideoObject:
|
||||
"""Transform the RunwayML video delete/cancel response."""
|
||||
response_data = raw_response.json()
|
||||
|
||||
video_obj = VideoObject(
|
||||
id=response_data.get("id", ""),
|
||||
object="video",
|
||||
status="cancelled",
|
||||
created_at=self._parse_runway_timestamp(response_data.get("createdAt")),
|
||||
) # type: ignore[arg-type]
|
||||
|
||||
return video_obj
|
||||
|
||||
def transform_video_status_retrieve_request(
|
||||
self,
|
||||
video_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform the RunwayML video status retrieve request.
|
||||
|
||||
RunwayML uses GET /v1/tasks/{task_id} to retrieve task status.
|
||||
"""
|
||||
original_video_id = extract_original_video_id(video_id)
|
||||
|
||||
# Construct the full URL for task status retrieval
|
||||
url = f"{api_base}/tasks/{original_video_id}"
|
||||
|
||||
# Empty dict for GET request (no body)
|
||||
data: Dict[str, Any] = {}
|
||||
|
||||
return url, data
|
||||
|
||||
def transform_video_status_retrieve_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
) -> VideoObject:
|
||||
"""
|
||||
Transform the RunwayML video status retrieve response.
|
||||
"""
|
||||
response_data = raw_response.json()
|
||||
|
||||
# Map RunwayML task response to VideoObject format
|
||||
video_data: Dict[str, Any] = {
|
||||
"id": response_data.get("id", ""),
|
||||
"object": "video",
|
||||
"status": self._map_runway_status(response_data.get("status", "pending")),
|
||||
"created_at": self._parse_runway_timestamp(response_data.get("createdAt")),
|
||||
}
|
||||
|
||||
# Add optional fields if present
|
||||
if "output" in response_data and response_data["output"]:
|
||||
video_data["output_url"] = response_data["output"][0] if isinstance(response_data["output"], list) else response_data["output"]
|
||||
|
||||
if "completedAt" in response_data:
|
||||
video_data["completed_at"] = self._parse_runway_timestamp(response_data.get("completedAt"))
|
||||
|
||||
if "progress" in response_data:
|
||||
video_data["progress"] = response_data["progress"]
|
||||
|
||||
if "failureCode" in response_data or "failure" in response_data:
|
||||
video_data["error"] = {
|
||||
"code": response_data.get("failureCode", "unknown"),
|
||||
"message": response_data.get("failure", "Video generation failed")
|
||||
}
|
||||
|
||||
video_obj = VideoObject(**video_data) # type: ignore[arg-type]
|
||||
|
||||
if custom_llm_provider and video_obj.id:
|
||||
video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, None)
|
||||
|
||||
return video_obj
|
||||
|
||||
def get_error_class(
|
||||
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
||||
) -> BaseLLMException:
|
||||
from ...base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
raise BaseLLMException(
|
||||
status_code=status_code,
|
||||
message=error_message,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
@ -24566,5 +24566,97 @@
|
||||
"1024x1792",
|
||||
"1792x1024"
|
||||
]
|
||||
},
|
||||
"runwayml/gen4_turbo": {
|
||||
"litellm_provider": "runwayml",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_video_per_second": 0.05,
|
||||
"source": "https://docs.dev.runwayml.com/guides/pricing/",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
],
|
||||
"supported_resolutions": [
|
||||
"1280x720",
|
||||
"720x1280"
|
||||
],
|
||||
"comment": "5 credits per second @ $0.01 per credit = $0.05 per second"
|
||||
},
|
||||
"runwayml/gen4_aleph": {
|
||||
"litellm_provider": "runwayml",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_video_per_second": 0.15,
|
||||
"source": "https://docs.dev.runwayml.com/guides/pricing/",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
],
|
||||
"supported_resolutions": [
|
||||
"1280x720",
|
||||
"720x1280"
|
||||
],
|
||||
"comment": "15 credits per second @ $0.01 per credit = $0.15 per second"
|
||||
},
|
||||
"runwayml/gen3a_turbo": {
|
||||
"litellm_provider": "runwayml",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_video_per_second": 0.05,
|
||||
"source": "https://docs.dev.runwayml.com/guides/pricing/",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
],
|
||||
"supported_resolutions": [
|
||||
"1280x720",
|
||||
"720x1280"
|
||||
],
|
||||
"comment": "5 credits per second @ $0.01 per credit = $0.05 per second"
|
||||
},
|
||||
"runwayml/gen4_image": {
|
||||
"litellm_provider": "runwayml",
|
||||
"mode": "image_generation",
|
||||
"input_cost_per_image": 0.05,
|
||||
"output_cost_per_image": 0.05,
|
||||
"source": "https://docs.dev.runwayml.com/guides/pricing/",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"image"
|
||||
],
|
||||
"supported_resolutions": [
|
||||
"1280x720",
|
||||
"1920x1080"
|
||||
],
|
||||
"comment": "5 credits per 720p image or 8 credits per 1080p image @ $0.01 per credit. Using 5 credits ($0.05) as base cost"
|
||||
},
|
||||
"runwayml/gen4_image_turbo": {
|
||||
"litellm_provider": "runwayml",
|
||||
"mode": "image_generation",
|
||||
"input_cost_per_image": 0.02,
|
||||
"output_cost_per_image": 0.02,
|
||||
"source": "https://docs.dev.runwayml.com/guides/pricing/",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"image"
|
||||
],
|
||||
"supported_resolutions": [
|
||||
"1280x720",
|
||||
"1920x1080"
|
||||
],
|
||||
"comment": "2 credits per image (any resolution) @ $0.01 per credit = $0.02 per image"
|
||||
}
|
||||
}
|
||||
|
||||
@ -14,6 +14,10 @@ model_list:
|
||||
model: bedrock/*
|
||||
custom_llm_provider: bedrock
|
||||
aws_region_name: us-west-2
|
||||
- model_name: runwayml/*
|
||||
litellm_params:
|
||||
model: runwayml/*
|
||||
|
||||
|
||||
|
||||
# like MCPs/vector stores
|
||||
|
||||
@ -2506,6 +2506,7 @@ class LlmProviders(str, Enum):
|
||||
ANTHROPIC_TEXT = "anthropic_text"
|
||||
BYTEZ = "bytez"
|
||||
REPLICATE = "replicate"
|
||||
RUNWAYML = "runwayml"
|
||||
HUGGINGFACE = "huggingface"
|
||||
TOGETHER_AI = "together_ai"
|
||||
OPENROUTER = "openrouter"
|
||||
|
||||
@ -7660,6 +7660,10 @@ class ProviderConfigManager:
|
||||
)
|
||||
|
||||
return VertexAIVideoConfig()
|
||||
elif LlmProviders.RUNWAYML == provider:
|
||||
from litellm.llms.runway.videos.transformation import RunwayMLVideoConfig
|
||||
|
||||
return RunwayMLVideoConfig()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
||||
@ -24566,5 +24566,97 @@
|
||||
"1024x1792",
|
||||
"1792x1024"
|
||||
]
|
||||
},
|
||||
"runwayml/gen4_turbo": {
|
||||
"litellm_provider": "runwayml",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_video_per_second": 0.05,
|
||||
"source": "https://docs.dev.runwayml.com/guides/pricing/",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
],
|
||||
"supported_resolutions": [
|
||||
"1280x720",
|
||||
"720x1280"
|
||||
],
|
||||
"comment": "5 credits per second @ $0.01 per credit = $0.05 per second"
|
||||
},
|
||||
"runwayml/gen4_aleph": {
|
||||
"litellm_provider": "runwayml",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_video_per_second": 0.15,
|
||||
"source": "https://docs.dev.runwayml.com/guides/pricing/",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
],
|
||||
"supported_resolutions": [
|
||||
"1280x720",
|
||||
"720x1280"
|
||||
],
|
||||
"comment": "15 credits per second @ $0.01 per credit = $0.15 per second"
|
||||
},
|
||||
"runwayml/gen3a_turbo": {
|
||||
"litellm_provider": "runwayml",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_video_per_second": 0.05,
|
||||
"source": "https://docs.dev.runwayml.com/guides/pricing/",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
],
|
||||
"supported_resolutions": [
|
||||
"1280x720",
|
||||
"720x1280"
|
||||
],
|
||||
"comment": "5 credits per second @ $0.01 per credit = $0.05 per second"
|
||||
},
|
||||
"runwayml/gen4_image": {
|
||||
"litellm_provider": "runwayml",
|
||||
"mode": "image_generation",
|
||||
"input_cost_per_image": 0.05,
|
||||
"output_cost_per_image": 0.05,
|
||||
"source": "https://docs.dev.runwayml.com/guides/pricing/",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"image"
|
||||
],
|
||||
"supported_resolutions": [
|
||||
"1280x720",
|
||||
"1920x1080"
|
||||
],
|
||||
"comment": "5 credits per 720p image or 8 credits per 1080p image @ $0.01 per credit. Using 5 credits ($0.05) as base cost"
|
||||
},
|
||||
"runwayml/gen4_image_turbo": {
|
||||
"litellm_provider": "runwayml",
|
||||
"mode": "image_generation",
|
||||
"input_cost_per_image": 0.02,
|
||||
"output_cost_per_image": 0.02,
|
||||
"source": "https://docs.dev.runwayml.com/guides/pricing/",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"image"
|
||||
],
|
||||
"supported_resolutions": [
|
||||
"1280x720",
|
||||
"1920x1080"
|
||||
],
|
||||
"comment": "2 credits per image (any resolution) @ $0.01 per credit = $0.02 per image"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1358,6 +1358,23 @@
|
||||
"rerank": false
|
||||
}
|
||||
},
|
||||
"runwayml": {
|
||||
"display_name": "RunwayML (`runwayml`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/runwayml/videos",
|
||||
"endpoints": {
|
||||
"chat_completions": false,
|
||||
"messages": false,
|
||||
"responses": false,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"moderations": false,
|
||||
"batches": false,
|
||||
"rerank": false,
|
||||
"video_generations": true
|
||||
}
|
||||
},
|
||||
"sagemaker_chat": {
|
||||
"display_name": "Sagemaker Chat (`sagemaker_chat`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/aws_sagemaker",
|
||||
|
||||
@ -0,0 +1,204 @@
|
||||
"""
|
||||
Tests for RunwayML video generation transformation.
|
||||
"""
|
||||
from unittest.mock import Mock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from litellm.llms.runway.videos.transformation import RunwayMLVideoConfig
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.videos.main import VideoObject
|
||||
|
||||
|
||||
class TestRunwayMLVideoTransformation:
|
||||
"""Test RunwayMLVideoConfig transformation class."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup test fixtures."""
|
||||
self.config = RunwayMLVideoConfig()
|
||||
self.mock_logging_obj = Mock()
|
||||
|
||||
def test_transform_video_create_request(self):
|
||||
"""Test video creation request validates URL and payload structure."""
|
||||
prompt = "A high quality demo video of litellm ai gateway"
|
||||
api_base = "https://api.dev.runwayml.com/v1"
|
||||
|
||||
data, files, url = self.config.transform_video_create_request(
|
||||
model="gen4_turbo",
|
||||
prompt=prompt,
|
||||
api_base=api_base,
|
||||
video_create_optional_request_params={
|
||||
"promptImage": "https://media.licdn.com/dms/image/v2/D4D0BAQFqOrIAJEgtLw/company-logo_200_200/company-logo_200_200/0/1714076049190/berri_ai_logo",
|
||||
"duration": 5,
|
||||
"ratio": "1280:720"
|
||||
},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={}
|
||||
)
|
||||
|
||||
# Validate payload structure
|
||||
assert data["model"] == "gen4_turbo"
|
||||
assert data["promptText"] == prompt
|
||||
assert data["promptImage"].startswith("https://")
|
||||
assert data["ratio"] == "1280:720"
|
||||
assert data["duration"] == 5
|
||||
assert files == []
|
||||
|
||||
# Validate URL has correct endpoint
|
||||
assert url == "https://api.dev.runwayml.com/v1/image_to_video"
|
||||
|
||||
def test_transform_video_status_with_timestamp_handling(self):
|
||||
"""Test status retrieval handles RunwayML's ISO 8601 timestamps correctly."""
|
||||
from litellm.types.videos.utils import encode_video_id_with_provider
|
||||
|
||||
# Test status request URL construction
|
||||
video_id = encode_video_id_with_provider(
|
||||
"63fd0f13-f29d-4e58-99d3-1cb9efa14a5b",
|
||||
"runwayml",
|
||||
"gen4_turbo"
|
||||
)
|
||||
api_base = "https://api.dev.runwayml.com/v1"
|
||||
|
||||
url, params = self.config.transform_video_status_retrieve_request(
|
||||
video_id=video_id,
|
||||
api_base=api_base,
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={}
|
||||
)
|
||||
|
||||
assert url == "https://api.dev.runwayml.com/v1/tasks/63fd0f13-f29d-4e58-99d3-1cb9efa14a5b"
|
||||
assert params == {}
|
||||
|
||||
# Test status response with ISO 8601 timestamp parsing
|
||||
mock_response = Mock(spec=httpx.Response)
|
||||
mock_response.json.return_value = {
|
||||
"id": "63fd0f13-f29d-4e58-99d3-1cb9efa14a5b",
|
||||
"createdAt": "2025-11-11T21:48:50.448Z",
|
||||
"status": "SUCCEEDED",
|
||||
"completedAt": "2025-11-11T21:50:15.123Z",
|
||||
"output": ["https://dnznrvs05pmza.cloudfront.net/video.mp4"],
|
||||
"progress": 100
|
||||
}
|
||||
|
||||
result = self.config.transform_video_status_retrieve_response(
|
||||
raw_response=mock_response,
|
||||
logging_obj=self.mock_logging_obj,
|
||||
custom_llm_provider="runwayml"
|
||||
)
|
||||
|
||||
assert isinstance(result, VideoObject)
|
||||
assert result.status == "completed"
|
||||
# Verify ISO 8601 timestamps are converted to Unix timestamps (integers)
|
||||
assert isinstance(result.created_at, int)
|
||||
assert result.created_at > 0
|
||||
assert isinstance(result.completed_at, int)
|
||||
assert result.completed_at > 0
|
||||
assert result.progress == 100
|
||||
|
||||
def test_transform_video_content_extraction(self):
|
||||
"""Test content retrieval extracts video URL from RunwayML response correctly."""
|
||||
from litellm.types.videos.utils import encode_video_id_with_provider
|
||||
|
||||
# Test content request URL
|
||||
video_id = encode_video_id_with_provider(
|
||||
"63fd0f13-f29d-4e58-99d3-1cb9efa14a5b",
|
||||
"runwayml",
|
||||
"gen4_turbo"
|
||||
)
|
||||
api_base = "https://api.dev.runwayml.com/v1"
|
||||
|
||||
url, params = self.config.transform_video_content_request(
|
||||
video_id=video_id,
|
||||
api_base=api_base,
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={}
|
||||
)
|
||||
|
||||
assert url == "https://api.dev.runwayml.com/v1/tasks/63fd0f13-f29d-4e58-99d3-1cb9efa14a5b"
|
||||
|
||||
# Test video URL extraction from response
|
||||
response_data = {
|
||||
"id": "test-id",
|
||||
"status": "SUCCEEDED",
|
||||
"output": ["https://dnznrvs05pmza.cloudfront.net/video.mp4"]
|
||||
}
|
||||
video_url = self.config._extract_video_url_from_response(response_data)
|
||||
assert video_url == "https://dnznrvs05pmza.cloudfront.net/video.mp4"
|
||||
|
||||
# Test error handling when video is still processing
|
||||
processing_response = {
|
||||
"id": "test-id",
|
||||
"status": "RUNNING",
|
||||
"output": None
|
||||
}
|
||||
with pytest.raises(ValueError, match="still processing"):
|
||||
self.config._extract_video_url_from_response(processing_response)
|
||||
|
||||
def test_full_video_workflow(self):
|
||||
"""Test complete video generation workflow from creation to status check."""
|
||||
config = RunwayMLVideoConfig()
|
||||
mock_logging_obj = Mock()
|
||||
|
||||
# Step 1: Create video
|
||||
prompt = "A high quality demo video of litellm ai gateway"
|
||||
api_base = "https://api.dev.runwayml.com/v1"
|
||||
data, files, url = config.transform_video_create_request(
|
||||
model="gen4_turbo",
|
||||
prompt=prompt,
|
||||
api_base=api_base,
|
||||
video_create_optional_request_params={
|
||||
"promptImage": "https://media.licdn.com/dms/image/v2/D4D0BAQFqOrIAJEgtLw/company-logo_200_200/company-logo_200_200/0/1714076049190/berri_ai_logo",
|
||||
"ratio": "1280:720",
|
||||
"duration": 5
|
||||
},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={}
|
||||
)
|
||||
|
||||
assert data["model"] == "gen4_turbo"
|
||||
assert url.endswith("/image_to_video")
|
||||
|
||||
# Step 2: Parse creation response
|
||||
mock_create_response = Mock(spec=httpx.Response)
|
||||
mock_create_response.json.return_value = {
|
||||
"id": "test-video-id-123",
|
||||
"createdAt": "2025-11-11T21:48:50.448Z",
|
||||
"status": "PENDING"
|
||||
}
|
||||
|
||||
video_obj = config.transform_video_create_response(
|
||||
model="gen4_turbo",
|
||||
raw_response=mock_create_response,
|
||||
logging_obj=mock_logging_obj,
|
||||
custom_llm_provider="runwayml",
|
||||
request_data=data
|
||||
)
|
||||
|
||||
assert video_obj.status == "queued"
|
||||
assert video_obj.id.startswith("video_")
|
||||
|
||||
# Step 3: Check completion status
|
||||
mock_status_response = Mock(spec=httpx.Response)
|
||||
mock_status_response.json.return_value = {
|
||||
"id": "test-video-id-123",
|
||||
"createdAt": "2025-11-11T21:48:50.448Z",
|
||||
"status": "SUCCEEDED",
|
||||
"completedAt": "2025-11-11T21:50:15.123Z",
|
||||
"output": ["https://dnznrvs05pmza.cloudfront.net/video.mp4"]
|
||||
}
|
||||
|
||||
status_obj = config.transform_video_status_retrieve_response(
|
||||
raw_response=mock_status_response,
|
||||
logging_obj=mock_logging_obj,
|
||||
custom_llm_provider="runwayml"
|
||||
)
|
||||
|
||||
assert status_obj.status == "completed"
|
||||
assert isinstance(status_obj.created_at, int)
|
||||
assert isinstance(status_obj.completed_at, int)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
Loading…
Reference in New Issue
Block a user