mirror of
https://github.com/usestrix/strix.git
synced 2026-08-22 19:09:37 +02:00
fix(context): over-estimate tokens when no tokenizer is available
The chars/4 fallback under-counts dense text (code, base64, CJK), which could let a summary request be packed past the real context window and get rejected. Use a conservative ~3-chars/token estimate instead so budget checks never under-count.
This commit is contained in:
@@ -68,10 +68,18 @@ def output_limit(model: str) -> int:
|
||||
|
||||
|
||||
def count_tokens(model: str, text: str) -> int:
|
||||
"""Token count for ``text`` under ``model`` (chars/4 fallback)."""
|
||||
"""Token count for ``text`` under ``model``.
|
||||
|
||||
LiteLLM's counter handles known tokenizers (and defaults to a tiktoken
|
||||
encoding otherwise). If it still can't count, fall back to a *conservative*
|
||||
estimate: token density varies, and dense text (code, base64, CJK) can run
|
||||
well under 4 chars/token, so we assume ~3 to over-estimate rather than
|
||||
under-estimate — an under-estimate would let a summary request be packed
|
||||
past the real context window and get rejected.
|
||||
"""
|
||||
if not text:
|
||||
return 0
|
||||
try:
|
||||
return int(litellm.token_counter(model=_lookup_key(model), text=text))
|
||||
except Exception: # noqa: BLE001 - tokenizer may be unavailable for some models.
|
||||
return len(text) // 4
|
||||
return -(-len(text) // 3)
|
||||
|
||||
Reference in New Issue
Block a user