ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #13374 from blahgeek/fix-count-special-token

Browse Source 
Fix token_counter with special token input
This commit is contained in:
Krish Dholakia 2025-08-26 23:19:26 -07:00 committed by GitHub
commit 71af7bcf9c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
litellm/litellm_core_utils/token_counter.py
View File

@ -529,7 +529,7 @@ def _get_count_function(
encoding = tiktoken.get_encoding("cl100k_base")
def count_tokens(text: str) -> int:
return len(encoding.encode(text))
return len(encoding.encode(text, disallowed_special=()))
else:
raise ValueError("Unsupported tokenizer type")

1
tests/test_litellm/litellm_core_utils/test_token_counter.py
View File

@ -451,6 +451,7 @@ def test_img_url_token_counter(img_url):
def test_token_encode_disallowed_special():
encode(model="gpt-3.5-turbo", text="Hello, world! <|endoftext|>")
token_counter(model="gpt-3.5-turbo", text="Hello, world! <|endoftext|>")
def test_token_counter():