forkless/strix
1
0
Fork 0
mirror of https://github.com/usestrix/strix.git synced 2026-08-22 02:58:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(context): over-estimate tokens when no tokenizer is available

Browse Source 
The chars/4 fallback under-counts dense text (code, base64, CJK), which
could let a summary request be packed past the real context window and get
rejected. Use a conservative ~3-chars/token estimate instead so budget
checks never under-count.
This commit is contained in:
Ahmed Allam
2026-07-26 00:04:13 +00:00
parent 16cbd516c8
commit 2163a66b78
2 changed files with 14 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_context_budget.py
+4 -2
View File
@@ -38,8 +38,10 @@ def test_count_tokens_fallback_on_error(monkeypatch: pytest.MonkeyPatch) -> None
raise RuntimeError("no tokenizer")
monkeypatch.setattr("strix.llm.context_budget.litellm.token_counter", _raise)
text = "x" * 400
assert context_budget.count_tokens("weird-model", text) == 100
# Conservative ~3-chars/token estimate, rounded up, so budgets never
# under-count when no tokenizer is available.
assert context_budget.count_tokens("weird-model", "x" * 400) == 134
assert context_budget.count_tokens("weird-model", "x") == 1
def test_count_tokens_empty_is_zero() -> None: