feat(llm): enable Bedrock/Anthropic prompt caching for Claude models

A Strix scan is a long multi-turn agentic loop that re-sends a large, STABLE
prefix every turn — the system prompt plus the tool schemas — while only the
conversation tail changes. Without a caching breakpoint that whole prefix is
re-tokenised and billed at full input rate on every turn; on Bedrock Claude
it's the single biggest lever on scan cost. Measured on a real scan: cache-read
went 0% -> 57% once these injection points are set (roughly halving input cost,
and the ratio climbs on longer scans where the stable prefix dominates more
turns).

LiteLLM already implements this end to end: when `cache_control_injection_points`
is present in the call kwargs its `AnthropicCacheControlHook` fires and emits the
provider-appropriate breakpoint (Anthropic `cache_control`; Bedrock Converse
`cachePoint`), honouring Anthropic's 4-breakpoint cap. `LitellmModel` forwards
`ModelSettings.extra_args` straight into `litellm.acompletion()`, so passing the
points there is all that's needed. We mark the two big stable segments (system
prompt + tool_config = 2 of 4 breakpoints, headroom left).

Deliberately kept at the LiteLLM-config layer rather than a general ModelSettings
caching flag — that's the direction the Agents SDK maintainer prescribed when
declining a native `cache_system_prompt` field
(openai/openai-agents-python#3008 / #3009): caching is a LiteLLM/provider
behaviour, and a ModelSettings flag would let strict OpenAI-compatible paths emit
non-standard cache_control parts. Gating on Claude keeps it a strict no-op for
every other provider (no injection points -> the hook never fires); only
Claude-family routes (Anthropic native, Bedrock, Vertex, OpenRouter -> Claude)
honour the marker.

Tests: parametrised, non-vacuous — Claude routes (bedrock/native/openrouter) get
the two injection points; non-Claude (gpt-5/gemini/o3) get extra_args=None.
This commit is contained in:
Sean Turner
2026-07-15 12:54:20 +01:00
parent 40f4e67320
commit d93a99a355
2 changed files with 77 additions and 0 deletions
+48
View File
@@ -141,9 +141,57 @@ def make_model_settings(
)
if force_required_tool_choice and _accepts_required_tool_choice(model_name):
model_settings = model_settings.resolve(ModelSettings(tool_choice="required"))
if _is_claude_model(model_name):
model_settings = model_settings.resolve(
ModelSettings(extra_args=_claude_prompt_cache_extra_args()),
)
return model_settings
def _is_claude_model(model_name: str) -> bool:
return "claude" in (model_name or "").strip().lower()
def _claude_prompt_cache_extra_args() -> dict[str, Any]:
"""Enable Anthropic/Bedrock prompt caching for Claude models via LiteLLM.
A Strix scan is a long, multi-turn agentic loop that re-sends a large,
STABLE prefix every turn — the system prompt plus the tool schemas — while
only the conversation tail changes. Without a caching breakpoint the whole
prefix is re-tokenised and billed at the full input rate on every turn; on
Bedrock Claude that is the single biggest lever on scan cost (measured here:
``cache-read 0% -> 57%`` on a real scan once these points are set).
LiteLLM already implements this end to end: when
``cache_control_injection_points`` is present in the call kwargs its
``AnthropicCacheControlHook`` fires and emits the provider-appropriate
breakpoint (Anthropic ``cache_control``; Bedrock Converse ``cachePoint``),
honouring Anthropic's 4-breakpoint cap. ``LitellmModel`` forwards
``ModelSettings.extra_args`` straight into ``litellm.acompletion()``, so
passing the injection points there is all that is required.
This is deliberately kept at the LiteLLM-config layer rather than a general
``ModelSettings`` caching flag: that is the direction the Agents SDK
maintainer prescribed when declining a native ``cache_system_prompt`` field
(openai/openai-agents-python#3008 / #3009) — caching is a LiteLLM/provider
behaviour and a ``ModelSettings`` flag would let strict OpenAI-compatible
paths emit non-standard ``cache_control`` parts. Gating on Claude keeps this
a no-op for every other provider (no injection points -> the hook never
fires), and only Claude-family routes (Anthropic native, Bedrock, Vertex,
OpenRouter -> Claude) honour the marker.
Two breakpoints on the stable prefix (2 of the 4 allowed), leaving headroom:
- the system prompt (``role: system``) — the largest repeated span
- the tool schemas (``tool_config``) — sizeable and identical every turn
"""
return {
"cache_control_injection_points": [
{"location": "message", "role": "system"},
{"location": "tool_config"},
],
}
def child_initial_input(
*,
name: str,
+29
View File
@@ -55,6 +55,35 @@ def test_child_initial_input_no_consecutive_same_role(parent_history: list[Any])
assert all(prev != nxt for prev, nxt in pairwise(roles))
def _cache_points(model_name: str) -> Any:
extra = make_model_settings(None, model_name=model_name).extra_args or {}
return extra.get("cache_control_injection_points")
@pytest.mark.parametrize(
"model_name",
[
"bedrock/global.anthropic.claude-opus-4-8",
"anthropic/claude-sonnet-4-5",
"openrouter/anthropic/claude-3.5-sonnet",
],
)
def test_make_model_settings_enables_prompt_cache_for_claude(model_name: str) -> None:
points = _cache_points(model_name)
assert points == [
{"location": "message", "role": "system"},
{"location": "tool_config"},
]
@pytest.mark.parametrize("model_name", ["gpt-5", "vertex_ai/gemini-2.5-pro", "openai/o3"])
def test_make_model_settings_no_prompt_cache_for_non_claude(model_name: str) -> None:
# No injection points for non-Claude models: the LiteLLM cache hook never
# fires, so this stays a strict no-op (won't emit cache_control to strict
# OpenAI-compatible endpoints).
assert make_model_settings(None, model_name=model_name).extra_args is None
def test_build_root_task_empty_config() -> None:
assert build_root_task({}) == ""