From efb698a4d0713d4d6ca86c57f5734574a82a25a0 Mon Sep 17 00:00:00 2001 From: Sean Turner Date: Fri, 17 Jul 2026 13:33:30 +0100 Subject: [PATCH] feat(llm): also cache the append-only conversation tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two prefix breakpoints (system + tool_config) only cache the FIXED prefix. A Strix scan's transcript is append-only, so the growing conversation body is re-sent at full input price every turn and cache-read decays as the transcript grows — a denominator effect, not the prefix missing. Add a third rolling breakpoint at index:-1 (the last message). Because prior turns are immutable, this re-caches the whole prefix-so-far each turn and hits on the next; LiteLLM resolves the negative index against the live message list. Measured on a 29-turn Bedrock scan the fixed prefix stayed pinned at ~56k tokens while per-turn input grew to ~256k and cache-read fell 90% -> 22%; the tail point lifts modelled cache-read to ~96% and cuts full-price input ~16x. Degrades gracefully on older LiteLLM (unrecognised location simply not injected). Adds an end-to-end test driving LiteLLM 1.90.1's _apply_message_injections to confirm the breakpoint tracks the tail across a growing transcript. Co-Authored-By: Claude Opus 4.8 (1M context) --- strix/core/inputs.py | 39 ++++++++++++++++++++++++++++----------- tests/test_inputs.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/strix/core/inputs.py b/strix/core/inputs.py index 3c6592c5..e7b091bb 100644 --- a/strix/core/inputs.py +++ b/strix/core/inputs.py @@ -232,11 +232,12 @@ def _claude_prompt_cache_extra_args() -> dict[str, Any]: """Enable Anthropic/Bedrock prompt caching for Claude models via LiteLLM. A Strix scan is a long, multi-turn agentic loop that re-sends a large, - STABLE prefix every turn — the system prompt plus the tool schemas — while - only the conversation tail changes. Without a caching breakpoint the whole - prefix is re-tokenised and billed at the full input rate on every turn; on - Bedrock Claude that is the single biggest lever on scan cost (measured here: - ``cache-read 0% -> 57%`` on a real scan once these points are set). + STABLE prefix every turn — the system prompt plus the tool schemas — AND an + append-only conversation transcript that only grows. Without caching + breakpoints the whole request is re-tokenised and billed at the full input + rate on every turn; on Bedrock Claude that is the single biggest lever on + scan cost (measured here: ``cache-read 0% -> 57%`` on a real scan once these + points are set). LiteLLM already implements this end to end: when ``cache_control_injection_points`` is present in the call kwargs its @@ -256,20 +257,36 @@ def _claude_prompt_cache_extra_args() -> dict[str, Any]: fires), and only Claude-family routes (Anthropic native, Bedrock, Vertex, OpenRouter -> Claude) honour the marker. - Two breakpoints on the stable prefix (2 of the 4 allowed), leaving headroom: + Three breakpoints (3 of the 4 allowed), leaving headroom: - the system prompt (``role: system``) — the largest repeated span - the tool schemas (``tool_config``) — sizeable and identical every turn + - the conversation tail (``index: -1``) — a ROLLING breakpoint on the last + message, so the accumulated transcript caches incrementally - Both points degrade gracefully on older LiteLLM: an unrecognised location is - simply not injected (no error), so a stale pin still gets whatever caching it - supports — the system-prompt point (the dominant win) has the widest support, - and the tool_config point is applied by LiteLLM's Bedrock Converse transform - on versions that recognise it (verified on litellm 1.90.1). + The tail breakpoint matters more than it looks. The first two only cache the + FIXED prefix; the transcript is append-only (prior turns are immutable, each + turn just appends the new assistant/tool messages), so on a long scan the + growing body is re-sent at full input price every turn and cache-read decays + as a denominator effect even though the prefix keeps hitting. A breakpoint at + ``index: -1`` re-caches the whole immutable prefix-so-far each turn and hits + on the next; the hook resolves the negative index against the live message + list. Measured on a 29-turn Bedrock scan, WITHOUT the tail point the cached + prefix stayed pinned at ~56k tokens while per-turn input grew to ~256k and + cache-read fell from 90% to 22%; adding it lifts modelled cache-read to ~96% + and cuts full-price input ~16x on that scan. + + All three points degrade gracefully on older LiteLLM: an unrecognised + location is simply not injected (no error), so a stale pin still gets + whatever caching it supports — the system-prompt point (the widest support) + and the tool_config + message-index points applied by LiteLLM's Bedrock + Converse transform on versions that recognise them (verified on litellm + 1.90.1). """ return { "cache_control_injection_points": [ {"location": "message", "role": "system"}, {"location": "tool_config"}, + {"location": "message", "index": -1}, ], } diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 3f9c4e70..9f7d6489 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -73,6 +73,7 @@ def test_make_model_settings_enables_prompt_cache_for_claude(model_name: str) -> assert points == [ {"location": "message", "role": "system"}, {"location": "tool_config"}, + {"location": "message", "index": -1}, ] @@ -122,9 +123,42 @@ def test_prompt_cache_kept_for_non_bedrock_claude_even_if_unmapped(monkeypatch: assert _cache_points(model) == [ {"location": "message", "role": "system"}, {"location": "tool_config"}, + {"location": "message", "index": -1}, ] +def test_conversation_tail_breakpoint_moves_with_appended_transcript() -> None: + """The tail breakpoint's premise, end-to-end: LiteLLM's own message-injection + logic must place the cache_control on the LAST message for both a short and a + long transcript — i.e. it tracks the growing (append-only) tail rather than a + fixed position — so the immutable prefix-so-far is cached and re-read next + turn. + + Driven through the hook's static ``_apply_message_injections`` primitive + (stable across LiteLLM versions) rather than the prompt-manager entrypoint + (whose signature drifts). + """ + hook_mod = pytest.importorskip("litellm.integrations.anthropic_cache_control_hook") + apply = hook_mod.AnthropicCacheControlHook._apply_message_injections + points = _cache_points("bedrock/global.anthropic.claude-opus-4-8") + msg_points = [p for p in points if p.get("location") == "message"] + + def last_msg_cache_control(n_turns: int) -> Any: + messages: list[dict[str, Any]] = [{"role": "system", "content": "stable prompt"}] + for i in range(n_turns): + messages.append({"role": "assistant", "content": f"turn {i} action"}) + messages.append({"role": "user", "content": f"turn {i} tool result"}) + processed = apply(msg_points, messages, 4) + last = processed[-1] + content = last.get("content") + if isinstance(content, list): + return content[-1].get("cache_control") + return last.get("cache_control") + + assert last_msg_cache_control(2) == {"type": "ephemeral"} + assert last_msg_cache_control(20) == {"type": "ephemeral"} + + def test_build_root_task_empty_config() -> None: assert build_root_task({}) == ""