diff --git a/.env.example b/.env.example index c6df78cafe..24c2b60841 100644 --- a/.env.example +++ b/.env.example @@ -20,10 +20,12 @@ REPLICATE_API_TOKEN = "" ANTHROPIC_API_KEY = "" # Infisical INFISICAL_TOKEN = "" +# Novita AI +NOVITA_API_KEY = "" # INFINITY INFINITY_API_KEY = "" # Development Configs LITELLM_MASTER_KEY = "sk-1234" DATABASE_URL = "postgresql://llmproxy:dbpassword9090@db:5432/litellm" -STORE_MODEL_IN_DB = "True" \ No newline at end of file +STORE_MODEL_IN_DB = "True" diff --git a/README.md b/README.md index 1c4e148443..3b9c111d75 100644 --- a/README.md +++ b/README.md @@ -332,6 +332,7 @@ curl 'http://0.0.0.0:4000/key/generate' \ | [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ | | | [FriendliAI](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ | ✅ | | | | [Galadriel](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ | ✅ | | | +| [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | ✅ | | | [**Read the Docs**](https://docs.litellm.ai/docs/) diff --git a/cookbook/LiteLLM_NovitaAI_Cookbook.ipynb b/cookbook/LiteLLM_NovitaAI_Cookbook.ipynb new file mode 100644 index 0000000000..8fa7d0b987 --- /dev/null +++ b/cookbook/LiteLLM_NovitaAI_Cookbook.ipynb @@ -0,0 +1,97 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "iFEmsVJI_2BR" + }, + "source": [ + "# LiteLLM NovitaAI Cookbook" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cBlUhCEP_xj4" + }, + "outputs": [], + "source": [ + "!pip install litellm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "p-MQqWOT_1a7" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['NOVITA_API_KEY'] = \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Ze8JqMqWAARO" + }, + "outputs": [], + "source": [ + "from litellm import completion\n", + "response = completion(\n", + " model=\"novita/deepseek/deepseek-r1\",\n", + " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", + ")\n", + "response" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-LnhELrnAM_J" + }, + "outputs": [], + "source": [ + "response = completion(\n", + " model=\"novita/deepseek/deepseek-r1\",\n", + " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", + ")\n", + "response" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dJBOUYdwCEn1" + }, + "outputs": [], + "source": [ + "response = completion(\n", + " model=\"mistralai/mistral-7b-instruct\",\n", + " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", + ")\n", + "response" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/cookbook/LiteLLM_OpenRouter.ipynb b/cookbook/LiteLLM_OpenRouter.ipynb index e0d03e1258..6444b23b29 100644 --- a/cookbook/LiteLLM_OpenRouter.ipynb +++ b/cookbook/LiteLLM_OpenRouter.ipynb @@ -1,27 +1,13 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, "cells": [ { "cell_type": "markdown", - "source": [ - "# LiteLLM OpenRouter Cookbook" - ], "metadata": { "id": "iFEmsVJI_2BR" - } + }, + "source": [ + "# LiteLLM OpenRouter Cookbook" + ] }, { "cell_type": "code", @@ -36,27 +22,20 @@ }, { "cell_type": "code", + "execution_count": 14, + "metadata": { + "id": "p-MQqWOT_1a7" + }, + "outputs": [], "source": [ "import os\n", "\n", "os.environ['OPENROUTER_API_KEY'] = \"\"" - ], - "metadata": { - "id": "p-MQqWOT_1a7" - }, - "execution_count": 14, - "outputs": [] + ] }, { "cell_type": "code", - "source": [ - "from litellm import completion\n", - "response = completion(\n", - " model=\"openrouter/google/palm-2-chat-bison\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ], + "execution_count": 11, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -64,10 +43,8 @@ "id": "Ze8JqMqWAARO", "outputId": "64f3e836-69fa-4f8e-fb35-088a913bbe98" }, - "execution_count": 11, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ " JSON: {\n", @@ -85,20 +62,23 @@ "}" ] }, + "execution_count": 11, "metadata": {}, - "execution_count": 11 + "output_type": "execute_result" } + ], + "source": [ + "from litellm import completion\n", + "response = completion(\n", + " model=\"openrouter/google/palm-2-chat-bison\",\n", + " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", + ")\n", + "response" ] }, { "cell_type": "code", - "source": [ - "response = completion(\n", - " model=\"openrouter/anthropic/claude-2\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ], + "execution_count": 12, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -106,10 +86,8 @@ "id": "-LnhELrnAM_J", "outputId": "d51c7ab7-d761-4bd1-f849-1534d9df4cd0" }, - "execution_count": 12, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ " JSON: {\n", @@ -128,20 +106,22 @@ "}" ] }, + "execution_count": 12, "metadata": {}, - "execution_count": 12 + "output_type": "execute_result" } + ], + "source": [ + "response = completion(\n", + " model=\"openrouter/anthropic/claude-2\",\n", + " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", + ")\n", + "response" ] }, { "cell_type": "code", - "source": [ - "response = completion(\n", - " model=\"openrouter/meta-llama/llama-2-70b-chat\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ], + "execution_count": 13, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -149,10 +129,8 @@ "id": "dJBOUYdwCEn1", "outputId": "ffa18679-ec15-4dad-fe2b-68665cdf36b0" }, - "execution_count": 13, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ " JSON: {\n", @@ -170,10 +148,32 @@ "}" ] }, + "execution_count": 13, "metadata": {}, - "execution_count": 13 + "output_type": "execute_result" } + ], + "source": [ + "response = completion(\n", + " model=\"openrouter/meta-llama/llama-2-70b-chat\",\n", + " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", + ")\n", + "response" ] } - ] -} \ No newline at end of file + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index a8aa79b8cb..d4ed0d2997 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -62,6 +62,7 @@ Use `litellm.get_supported_openai_params()` for an updated list of params for ea |Databricks| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | | |ClarifAI| ✅ | ✅ | ✅ | |✅ | ✅ | | | | | | | | | | | |Github| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ |✅ (model dependent)|✅ (model dependent)| | | +|Novita AI| ✅ | ✅ | | ✅ | ✅ | ✅ | | ✅ | ✅ | ✅ | ✅ | | | ✅ | | | | | | | | :::note By default, LiteLLM raises an exception if the openai param being passed in isn't supported. diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index 9e4d76b89c..58cabc81b4 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -208,6 +208,22 @@ response = completion( ) ``` + + + +```python +from litellm import completion +import os + +## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key +os.environ["NOVITA_API_KEY"] = "novita-api-key" + +response = completion( + model="novita/deepseek/deepseek-r1", + messages=[{ "content": "Hello, how are you?","role": "user"}] +) +``` + @@ -411,6 +427,23 @@ response = completion( ) ``` + + + +```python +from litellm import completion +import os + +## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key +os.environ["NOVITA_API_KEY"] = "novita_api_key" + +response = completion( + model="novita/deepseek/deepseek-r1", + messages = [{ "content": "Hello, how are you?","role": "user"}], + stream=True, +) +``` + diff --git a/docs/my-website/docs/providers/novita.md b/docs/my-website/docs/providers/novita.md new file mode 100644 index 0000000000..88deddd141 --- /dev/null +++ b/docs/my-website/docs/providers/novita.md @@ -0,0 +1,39 @@ +# Novita AI +LiteLLM supports all models from [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) + +## Usage +```python +import os +from litellm import completion +os.environ["NOVITA_API_KEY"] = "" + +response = completion( + model="meta-llama/llama-3.3-70b-instruct", + messages=messages, + ) +``` + +## Novita AI Completion Models + +🚨 LiteLLM supports ALL Novita AI models, send `model=novita/` to send it to Novita AI. See all Novita AI models [here](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) + +| Model Name | Function Call | +|---------------------------|-----------------------------------------------------| +| novita/deepseek/deepseek-r1 | `completion('novita/deepseek/deepseek-r1', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/deepseek/deepseek_v3 | `completion('novita/deepseek/deepseek_v3', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3.3-70b-instruct | `completion('novita/meta-llama/llama-3.3-70b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3.1-8b-instruct | `completion('novita/meta-llama/llama-3.1-8b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3.1-8b-instruct-max | `completion('novita/meta-llama/llama-3.1-8b-instruct-max', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3.1-70b-instruct | `completion('novita/meta-llama/llama-3.1-70b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3-8b-instruct | `completion('novita/meta-llama/llama-3-8b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3-70b-instruct | `completion('novita/meta-llama/llama-3-70b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3.2-1b-instruct | `completion('novita/meta-llama/llama-3.2-1b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3.2-11b-vision-instruct | `completion('novita/meta-llama/llama-3.2-11b-vision-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3.2-3b-instruct | `completion('novita/meta-llama/llama-3.2-3b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/gryphe/mythomax-l2-13b | `completion('novita/gryphe/mythomax-l2-13b', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/google/gemma-2-9b-it | `completion('novita/google/gemma-2-9b-it', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/mistralai/mistral-nemo | `completion('novita/mistralai/mistral-nemo', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/mistralai/mistral-7b-instruct | `completion('novita/mistralai/mistral-7b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/qwen/qwen-2.5-72b-instruct | `completion('novita/qwen/qwen-2.5-72b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/qwen/qwen-2-vl-72b-instruct | `completion('novita/qwen/qwen-2-vl-72b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 8d063b878d..ccc2f6ccc1 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -354,6 +354,7 @@ const sidebars = { "providers/nlp_cloud", "providers/replicate", "providers/togetherai", + "providers/novita", "providers/voyage", "providers/jina_ai", "providers/aleph_alpha", diff --git a/docs/my-website/src/pages/completion/input.md b/docs/my-website/src/pages/completion/input.md index 86546bbbae..ff9a3f0f0a 100644 --- a/docs/my-website/src/pages/completion/input.md +++ b/docs/my-website/src/pages/completion/input.md @@ -1,6 +1,6 @@ # Completion Function - completion() The Input params are **exactly the same** as the -OpenAI Create chat completion, and let you call **Azure OpenAI, Anthropic, Cohere, Replicate, OpenRouter** models in the same format. +OpenAI Create chat completion, and let you call **Azure OpenAI, Anthropic, Cohere, Replicate, OpenRouter, Novita AI** models in the same format. In addition, liteLLM allows you to pass in the following **Optional** liteLLM args: `force_timeout`, `azure`, `logger_fn`, `verbose` diff --git a/docs/my-website/src/pages/completion/supported.md b/docs/my-website/src/pages/completion/supported.md index 2599353aa3..097af2bb4c 100644 --- a/docs/my-website/src/pages/completion/supported.md +++ b/docs/my-website/src/pages/completion/supported.md @@ -70,4 +70,28 @@ All the text models from [OpenRouter](https://openrouter.ai/docs) are supported | google/palm-2-chat-bison | `completion('google/palm-2-chat-bison', messages)` | `os.environ['OR_SITE_URL']`,`os.environ['OR_APP_NAME']`,`os.environ['OR_API_KEY']` | | google/palm-2-codechat-bison | `completion('google/palm-2-codechat-bison', messages)` | `os.environ['OR_SITE_URL']`,`os.environ['OR_APP_NAME']`,`os.environ['OR_API_KEY']` | | meta-llama/llama-2-13b-chat | `completion('meta-llama/llama-2-13b-chat', messages)` | `os.environ['OR_SITE_URL']`,`os.environ['OR_APP_NAME']`,`os.environ['OR_API_KEY']` | -| meta-llama/llama-2-70b-chat | `completion('meta-llama/llama-2-70b-chat', messages)` | `os.environ['OR_SITE_URL']`,`os.environ['OR_APP_NAME']`,`os.environ['OR_API_KEY']` | \ No newline at end of file +| meta-llama/llama-2-70b-chat | `completion('meta-llama/llama-2-70b-chat', messages)` | `os.environ['OR_SITE_URL']`,`os.environ['OR_APP_NAME']`,`os.environ['OR_API_KEY']` | + +## Novita AI Completion Models + +🚨 LiteLLM supports ALL Novita AI models, send `model=novita/` to send it to Novita AI. See all Novita AI models [here](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) + +| Model Name | Function Call | Required OS Variables | +|------------------|--------------------------------------------|--------------------------------------| +| novita/deepseek/deepseek-r1 | `completion('novita/deepseek/deepseek-r1', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/deepseek/deepseek_v3 | `completion('novita/deepseek/deepseek_v3', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3.3-70b-instruct | `completion('novita/meta-llama/llama-3.3-70b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3.1-8b-instruct | `completion('novita/meta-llama/llama-3.1-8b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3.1-8b-instruct-max | `completion('novita/meta-llama/llama-3.1-8b-instruct-max', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3.1-70b-instruct | `completion('novita/meta-llama/llama-3.1-70b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3-8b-instruct | `completion('novita/meta-llama/llama-3-8b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3-70b-instruct | `completion('novita/meta-llama/llama-3-70b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3.2-1b-instruct | `completion('novita/meta-llama/llama-3.2-1b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3.2-11b-vision-instruct | `completion('novita/meta-llama/llama-3.2-11b-vision-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/meta-llama/llama-3.2-3b-instruct | `completion('novita/meta-llama/llama-3.2-3b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/gryphe/mythomax-l2-13b | `completion('novita/gryphe/mythomax-l2-13b', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/google/gemma-2-9b-it | `completion('novita/google/gemma-2-9b-it', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/mistralai/mistral-nemo | `completion('novita/mistralai/mistral-nemo', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/mistralai/mistral-7b-instruct | `completion('novita/mistralai/mistral-7b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/qwen/qwen-2.5-72b-instruct | `completion('novita/qwen/qwen-2.5-72b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | +| novita/qwen/qwen-2-vl-72b-instruct | `completion('novita/qwen/qwen-2-vl-72b-instruct', messages)` | `os.environ['NOVITA_API_KEY']` | \ No newline at end of file diff --git a/docs/my-website/src/pages/index.md b/docs/my-website/src/pages/index.md index 4a2e5203e3..2c89d28a62 100644 --- a/docs/my-website/src/pages/index.md +++ b/docs/my-website/src/pages/index.md @@ -194,6 +194,22 @@ response = completion( ) ``` + + + +```python +from litellm import completion +import os + +## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key +os.environ["NOVITA_API_KEY"] = "novita-api-key" + +response = completion( + model="novita/deepseek/deepseek-r1", + messages=[{ "content": "Hello, how are you?","role": "user"}] +) +``` + @@ -347,7 +363,23 @@ response = completion( ``` + +```python +from litellm import completion +import os + +## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key +os.environ["NOVITA_API_KEY"] = "novita_api_key" + +response = completion( + model="novita/deepseek/deepseek-r1", + messages = [{ "content": "Hello, how are you?","role": "user"}], + stream=True, +) +``` + + ### Exception handling diff --git a/litellm/__init__.py b/litellm/__init__.py index f1b481612c..65cb61749c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -199,6 +199,7 @@ baseten_key: Optional[str] = None llama_api_key: Optional[str] = None aleph_alpha_key: Optional[str] = None nlp_cloud_key: Optional[str] = None +novita_api_key: Optional[str] = None snowflake_key: Optional[str] = None common_cloud_provider_auth_params: dict = { "params": ["project", "region_name", "token"], @@ -440,12 +441,12 @@ anyscale_models: List = [] cerebras_models: List = [] galadriel_models: List = [] sambanova_models: List = [] +novita_models: List = [] assemblyai_models: List = [] snowflake_models: List = [] llama_models: List = [] nscale_models: List = [] - def is_bedrock_pricing_only_model(key: str) -> bool: """ Excludes keys with the pattern 'bedrock//'. These are in the model_prices_and_context_window.json file for pricing purposes only. @@ -599,6 +600,8 @@ def add_known_models(): galadriel_models.append(key) elif value.get("litellm_provider") == "sambanova_models": sambanova_models.append(key) + elif value.get("litellm_provider") == "novita": + novita_models.append(key) elif value.get("litellm_provider") == "assemblyai": assemblyai_models.append(key) elif value.get("litellm_provider") == "jina_ai": @@ -678,6 +681,7 @@ model_list = ( + galadriel_models + sambanova_models + azure_text_models + + novita_models + assemblyai_models + jina_ai_models + snowflake_models @@ -737,6 +741,7 @@ models_by_provider: dict = { "cerebras": cerebras_models, "galadriel": galadriel_models, "sambanova": sambanova_models, + "novita": novita_models, "assemblyai": assemblyai_models, "jina_ai": jina_ai_models, "snowflake": snowflake_models, @@ -878,6 +883,7 @@ from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformat from .llms.together_ai.chat import TogetherAIConfig from .llms.together_ai.completion.transformation import TogetherAITextCompletionConfig from .llms.cloudflare.chat.transformation import CloudflareChatConfig +from .llms.novita.chat.transformation import NovitaConfig from .llms.deprecated_providers.palm import ( PalmConfig, ) # here to prevent breaking changes diff --git a/litellm/constants.py b/litellm/constants.py index 329118c302..6ef284bc31 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -161,6 +161,7 @@ LITELLM_CHAT_PROVIDERS = [ "llamafile", "lm_studio", "galadriel", + "novita", "meta_llama", "nscale", ] @@ -255,6 +256,7 @@ openai_compatible_providers: List = [ "llamafile", "lm_studio", "galadriel", + "novita", "meta_llama", "nscale", ] diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 6d03964f2b..7543d1722c 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -604,6 +604,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 or "https://api.galadriel.com/v1" ) # type: ignore dynamic_api_key = api_key or get_secret_str("GALADRIEL_API_KEY") + elif custom_llm_provider == "novita": + api_base = ( + api_base + or get_secret("NOVITA_API_BASE") + or "https://api.novita.ai/v3/openai" + ) # type: ignore + dynamic_api_key = api_key or get_secret_str("NOVITA_API_KEY") elif custom_llm_provider == "snowflake": api_base = ( api_base diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index 11981e627b..043444cfc6 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -155,6 +155,8 @@ def get_supported_openai_params( # noqa: PLR0915 return litellm.GoogleAIStudioGeminiConfig().get_supported_openai_params( model=model ) + elif custom_llm_provider == "novita": + return litellm.NovitaConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta": if request_type == "chat_completion": if model.startswith("mistral"): diff --git a/litellm/llms/novita/chat/transformation.py b/litellm/llms/novita/chat/transformation.py new file mode 100644 index 0000000000..c05d2d7b2c --- /dev/null +++ b/litellm/llms/novita/chat/transformation.py @@ -0,0 +1,33 @@ +""" +Support for OpenAI's `/v1/chat/completions` endpoint. + +Calls done in OpenAI/openai.py as Novita AI is openai-compatible. + +Docs: https://novita.ai/docs/guides/llm-api +""" + +from typing import List, Optional + +from ....types.llms.openai import AllMessageValues +from ...openai.chat.gpt_transformation import OpenAIGPTConfig + + +class NovitaConfig(OpenAIGPTConfig): + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + if api_key is None: + raise ValueError( + "Missing Novita AI API Key - A call is being made to novita but no key is set either in the environment variables or via params" + ) + headers["Authorization"] = f"Bearer {api_key}" + headers["Content-Type"] = "application/json" + headers["X-Novita-Source"] = "litellm" + return headers diff --git a/litellm/main.py b/litellm/main.py index e09c8b3418..86f13f3c56 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3328,7 +3328,6 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse: response = init_response elif asyncio.iscoroutine(init_response): response = await init_response # type: ignore - if ( response is not None and isinstance(response, EmbeddingResponse) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index bcc4bf7ab8..63c4835ae8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2151,6 +2151,7 @@ class LlmProviders(str, Enum): GALADRIEL = "galadriel" INFINITY = "infinity" DEEPGRAM = "deepgram" + NOVITA = "novita" AIOHTTP_OPENAI = "aiohttp_openai" LANGFUSE = "langfuse" HUMANLOOP = "humanloop" diff --git a/litellm/utils.py b/litellm/utils.py index b2e9caf1c4..bc1dc08cd9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2091,6 +2091,9 @@ def register_model(model_cost: Union[str, dict]): # noqa: PLR0915 elif value.get("litellm_provider") == "bedrock": if key not in litellm.bedrock_models: litellm.bedrock_models.append(key) + elif value.get("litellm_provider") == "novita": + if key not in litellm.novita_models: + litellm.novita_models.append(key) return model_cost @@ -4943,6 +4946,11 @@ def validate_environment( # noqa: PLR0915 else: missing_keys.append("CLOUDFLARE_API_KEY") missing_keys.append("CLOUDFLARE_API_BASE") + elif custom_llm_provider == "novita": + if "NOVITA_API_KEY" in os.environ: + keys_in_environment = True + else: + missing_keys.append("NOVITA_API_KEY") else: ## openai - chatcompletion + text completion if ( @@ -5025,6 +5033,11 @@ def validate_environment( # noqa: PLR0915 keys_in_environment = True else: missing_keys.append("NLP_CLOUD_API_KEY") + elif model in litellm.novita_models: + if "NOVITA_API_KEY" in os.environ: + keys_in_environment = True + else: + missing_keys.append("NOVITA_API_KEY") if api_key is not None: new_missing_keys = [] @@ -6366,6 +6379,8 @@ class ProviderConfigManager: return litellm.TritonConfig() elif litellm.LlmProviders.PETALS == provider: return litellm.PetalsConfig() + elif litellm.LlmProviders.NOVITA == provider: + return litellm.NovitaConfig() elif litellm.LlmProviders.BEDROCK == provider: bedrock_route = BedrockModelInfo.get_bedrock_route(model) bedrock_invoke_provider = litellm.BedrockLLM.get_bedrock_invoke_provider( diff --git a/poetry.lock b/poetry.lock index a2abc92831..c80d13a4b3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3808,7 +3808,7 @@ crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] name = "six" version = "1.17.0" description = "Python 2 and 3 compatibility utilities" -optional = true +optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] markers = "extra == \"extra-proxy\" or extra == \"proxy\"" diff --git a/tests/litellm/llms/novita/chat/test_transformation.py b/tests/litellm/llms/novita/chat/test_transformation.py new file mode 100644 index 0000000000..5216ea9488 --- /dev/null +++ b/tests/litellm/llms/novita/chat/test_transformation.py @@ -0,0 +1,69 @@ +""" +Unit tests for Novita AI configuration. + +These tests validate the NovitaConfig class which extends OpenAIGPTConfig. +Novita AI is an OpenAI-compatible provider with a few customizations. +""" + +import os +import sys +from typing import Dict, List, Optional +from unittest.mock import patch + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.novita.chat.transformation import NovitaConfig + + +class TestNovitaConfig: + """Test class for NovitaConfig functionality""" + + def test_validate_environment(self): + """Test that validate_environment adds correct headers""" + config = NovitaConfig() + headers = {} + api_key = "fake-novita-key" + + result = config.validate_environment( + headers=headers, + model="novita/meta-llama/llama-3.3-70b-instruct", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=api_key, + api_base="https://api.novita.ai/v3/openai" + ) + + # Verify headers + assert result["Authorization"] == f"Bearer {api_key}" + assert result["Content-Type"] == "application/json" + assert result["X-Novita-Source"] == "litellm" + + def test_missing_api_key(self): + """Test error handling when API key is missing""" + config = NovitaConfig() + + with pytest.raises(ValueError) as excinfo: + config.validate_environment( + headers={}, + model="novita/meta-llama/llama-3.3-70b-instruct", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=None, + api_base="https://api.novita.ai/v3/openai" + ) + + assert "Missing Novita AI API Key" in str(excinfo.value) + + def test_inheritance(self): + """Test proper inheritance from OpenAIGPTConfig""" + config = NovitaConfig() + + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + assert isinstance(config, OpenAIGPTConfig) + assert hasattr(config, "get_supported_openai_params") \ No newline at end of file diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 321b39f9c6..ef7086ac11 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -4394,6 +4394,77 @@ def test_humanloop_completion(monkeypatch): messages=[{"role": "user", "content": "Tell me a joke."}], ) +def test_completion_novita_ai(): + litellm.set_verbose = True + messages = [ + {"role": "system", "content": "You're a good bot"}, + { + "role": "user", + "content": "Hey", + }, + ] + + from openai import OpenAI + openai_client = OpenAI(api_key="fake-key") + + with patch.object( + openai_client.chat.completions, "create", new=MagicMock() + ) as mock_call: + try: + completion( + model="novita/meta-llama/llama-3.3-70b-instruct", + messages=messages, + client=openai_client, + api_base="https://api.novita.ai/v3/openai", + ) + + mock_call.assert_called_once() + + # Verify model is passed correctly + assert mock_call.call_args.kwargs["model"] == "meta-llama/llama-3.3-70b-instruct" + # Verify messages are passed correctly + assert mock_call.call_args.kwargs["messages"] == messages + + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + +@pytest.mark.parametrize( + "api_key", ["my-bad-api-key"] +) +def test_completion_novita_ai_dynamic_params(api_key): + try: + litellm.set_verbose = True + messages = [ + {"role": "system", "content": "You're a good bot"}, + { + "role": "user", + "content": "Hey", + }, + ] + + from openai import OpenAI + openai_client = OpenAI(api_key="fake-key") + + with patch.object( + openai_client.chat.completions, "create", side_effect=Exception("Invalid API key") + ) as mock_call: + try: + completion( + model="novita/meta-llama/llama-3.3-70b-instruct", + messages=messages, + api_key=api_key, + client=openai_client, + api_base="https://api.novita.ai/v3/openai", + ) + pytest.fail(f"This call should have failed!") + except Exception as e: + # This should fail with the mocked exception + assert "Invalid API key" in str(e) + + mock_call.assert_called_once() + except Exception as e: + pytest.fail(f"Unexpected error: {e}") def test_deepseek_reasoning_content_completion(): try: