diff --git a/strix/llm/compaction.py b/strix/llm/compaction.py index 59eb1bf2..85c008ec 100644 --- a/strix/llm/compaction.py +++ b/strix/llm/compaction.py @@ -192,9 +192,18 @@ def _fit_to_tokens(model: str, text: str, max_tokens: int) -> str: return candidate +def _summary_output_tokens(model: str) -> int: + """Summary output allowance, capped at the model's own output limit. + + A configured ``summary_max_tokens`` above the model's cap would make the + provider reject the summary request, so compaction would silently fail and + leave the overflowing session unchanged. + """ + return min(load_settings().context.summary_max_tokens, output_limit(model)) + + def _summary_input_budget(model: str, previous: str | None) -> int: """Token room left for the head after instructions and the summary output.""" - context = load_settings().context overhead = count_tokens(model, _SUMMARY_INSTRUCTIONS) if previous: overhead += count_tokens(model, previous) @@ -202,7 +211,7 @@ def _summary_input_budget(model: str, previous: str | None) -> int: # the update instructions, etc.) that is not part of ``overhead``. Never # floor above the actual room: doing so would let the summary request # itself overflow a small window (and then compaction silently fails). - room = context_window(model) - context.summary_max_tokens - overhead - 256 + room = context_window(model) - _summary_output_tokens(model) - overhead - 256 return max(0, room) @@ -300,7 +309,7 @@ async def maybe_compact( summary = await _summarize( model, _build_summary_prompt(serialized_head, previous), - context.summary_max_tokens, + _summary_output_tokens(model), ) if summary is None: return False diff --git a/strix/llm/context_budget.py b/strix/llm/context_budget.py index 6bd805e3..bfc50876 100644 --- a/strix/llm/context_budget.py +++ b/strix/llm/context_budget.py @@ -71,10 +71,11 @@ def count_tokens(model: str, text: str) -> int: """Token count for ``text`` under ``model``. LiteLLM's counter handles known tokenizers (and defaults to a tiktoken - encoding otherwise). If it still can't count, fall back to a *conservative* - estimate: token density varies, and dense text (code, base64, CJK) can run - well under 4 chars/token, so we assume ~3 to over-estimate rather than - under-estimate — an under-estimate would let a summary request be packed + encoding otherwise). If it still can't count, fall back to the UTF-8 byte + length as a guaranteed upper bound: byte-level BPE tokenizers (used by every + major provider) emit at least one byte per token, so token count can never + exceed the byte count. Over-counting is safe here — it makes budget checks + conservative — whereas any under-count could let a summary request be packed past the real context window and get rejected. """ if not text: @@ -82,4 +83,4 @@ def count_tokens(model: str, text: str) -> int: try: return int(litellm.token_counter(model=_lookup_key(model), text=text)) except Exception: # noqa: BLE001 - tokenizer may be unavailable for some models. - return -(-len(text) // 3) + return len(text.encode("utf-8")) diff --git a/tests/test_compaction.py b/tests/test_compaction.py index 01d5d5ac..fcaf16a8 100644 --- a/tests/test_compaction.py +++ b/tests/test_compaction.py @@ -190,6 +190,19 @@ def test_fit_to_tokens_truncates_oversized_text(monkeypatch: pytest.MonkeyPatch) assert compaction._fit_to_tokens("m", "short", 500) == "short" +def test_summary_output_tokens_capped_at_model_limit(monkeypatch: pytest.MonkeyPatch) -> None: + context = ContextSettings() + monkeypatch.setattr(compaction, "load_settings", lambda: SimpleNamespace(context=context)) + + monkeypatch.setattr(compaction, "output_limit", lambda _m: 1_000) + context.summary_max_tokens = 4_096 + # Configured allowance above the model cap is clamped down to the cap. + assert compaction._summary_output_tokens("m") == 1_000 + # Below the cap, the configured value is used unchanged. + context.summary_max_tokens = 500 + assert compaction._summary_output_tokens("m") == 500 + + @pytest.mark.asyncio async def test_maybe_compact_bounds_summary_prompt(monkeypatch: pytest.MonkeyPatch) -> None: # A tiny window with a huge head must not send an oversized summary request. diff --git a/tests/test_context_budget.py b/tests/test_context_budget.py index b2032137..f93bdce7 100644 --- a/tests/test_context_budget.py +++ b/tests/test_context_budget.py @@ -38,10 +38,11 @@ def test_count_tokens_fallback_on_error(monkeypatch: pytest.MonkeyPatch) -> None raise RuntimeError("no tokenizer") monkeypatch.setattr("strix.llm.context_budget.litellm.token_counter", _raise) - # Conservative ~3-chars/token estimate, rounded up, so budgets never + # UTF-8 byte length is a guaranteed upper bound on tokens, so budgets never # under-count when no tokenizer is available. - assert context_budget.count_tokens("weird-model", "x" * 400) == 134 - assert context_budget.count_tokens("weird-model", "x") == 1 + assert context_budget.count_tokens("weird-model", "x" * 400) == 400 + # Multibyte text: bytes (not chars) bound the token count. + assert context_budget.count_tokens("weird-model", "😀" * 10) == 40 def test_count_tokens_empty_is_zero() -> None: