mirror of
https://github.com/usestrix/strix.git
synced 2026-08-16 17:27:26 +02:00
feat(llm): enable Bedrock/Anthropic prompt caching for Claude models
A Strix scan is a long multi-turn agentic loop that re-sends a large, STABLE prefix every turn — the system prompt plus the tool schemas — while only the conversation tail changes. Without a caching breakpoint that whole prefix is re-tokenised and billed at full input rate on every turn; on Bedrock Claude it's the single biggest lever on scan cost. Measured on a real scan: cache-read went 0% -> 57% once these injection points are set (roughly halving input cost, and the ratio climbs on longer scans where the stable prefix dominates more turns). LiteLLM already implements this end to end: when `cache_control_injection_points` is present in the call kwargs its `AnthropicCacheControlHook` fires and emits the provider-appropriate breakpoint (Anthropic `cache_control`; Bedrock Converse `cachePoint`), honouring Anthropic's 4-breakpoint cap. `LitellmModel` forwards `ModelSettings.extra_args` straight into `litellm.acompletion()`, so passing the points there is all that's needed. We mark the two big stable segments (system prompt + tool_config = 2 of 4 breakpoints, headroom left). Deliberately kept at the LiteLLM-config layer rather than a general ModelSettings caching flag — that's the direction the Agents SDK maintainer prescribed when declining a native `cache_system_prompt` field (openai/openai-agents-python#3008 / #3009): caching is a LiteLLM/provider behaviour, and a ModelSettings flag would let strict OpenAI-compatible paths emit non-standard cache_control parts. Gating on Claude keeps it a strict no-op for every other provider (no injection points -> the hook never fires); only Claude-family routes (Anthropic native, Bedrock, Vertex, OpenRouter -> Claude) honour the marker. Tests: parametrised, non-vacuous — Claude routes (bedrock/native/openrouter) get the two injection points; non-Claude (gpt-5/gemini/o3) get extra_args=None.
This commit is contained in:
@@ -141,9 +141,57 @@ def make_model_settings(
|
||||
)
|
||||
if force_required_tool_choice and _accepts_required_tool_choice(model_name):
|
||||
model_settings = model_settings.resolve(ModelSettings(tool_choice="required"))
|
||||
if _is_claude_model(model_name):
|
||||
model_settings = model_settings.resolve(
|
||||
ModelSettings(extra_args=_claude_prompt_cache_extra_args()),
|
||||
)
|
||||
return model_settings
|
||||
|
||||
|
||||
def _is_claude_model(model_name: str) -> bool:
|
||||
return "claude" in (model_name or "").strip().lower()
|
||||
|
||||
|
||||
def _claude_prompt_cache_extra_args() -> dict[str, Any]:
|
||||
"""Enable Anthropic/Bedrock prompt caching for Claude models via LiteLLM.
|
||||
|
||||
A Strix scan is a long, multi-turn agentic loop that re-sends a large,
|
||||
STABLE prefix every turn — the system prompt plus the tool schemas — while
|
||||
only the conversation tail changes. Without a caching breakpoint the whole
|
||||
prefix is re-tokenised and billed at the full input rate on every turn; on
|
||||
Bedrock Claude that is the single biggest lever on scan cost (measured here:
|
||||
``cache-read 0% -> 57%`` on a real scan once these points are set).
|
||||
|
||||
LiteLLM already implements this end to end: when
|
||||
``cache_control_injection_points`` is present in the call kwargs its
|
||||
``AnthropicCacheControlHook`` fires and emits the provider-appropriate
|
||||
breakpoint (Anthropic ``cache_control``; Bedrock Converse ``cachePoint``),
|
||||
honouring Anthropic's 4-breakpoint cap. ``LitellmModel`` forwards
|
||||
``ModelSettings.extra_args`` straight into ``litellm.acompletion()``, so
|
||||
passing the injection points there is all that is required.
|
||||
|
||||
This is deliberately kept at the LiteLLM-config layer rather than a general
|
||||
``ModelSettings`` caching flag: that is the direction the Agents SDK
|
||||
maintainer prescribed when declining a native ``cache_system_prompt`` field
|
||||
(openai/openai-agents-python#3008 / #3009) — caching is a LiteLLM/provider
|
||||
behaviour and a ``ModelSettings`` flag would let strict OpenAI-compatible
|
||||
paths emit non-standard ``cache_control`` parts. Gating on Claude keeps this
|
||||
a no-op for every other provider (no injection points -> the hook never
|
||||
fires), and only Claude-family routes (Anthropic native, Bedrock, Vertex,
|
||||
OpenRouter -> Claude) honour the marker.
|
||||
|
||||
Two breakpoints on the stable prefix (2 of the 4 allowed), leaving headroom:
|
||||
- the system prompt (``role: system``) — the largest repeated span
|
||||
- the tool schemas (``tool_config``) — sizeable and identical every turn
|
||||
"""
|
||||
return {
|
||||
"cache_control_injection_points": [
|
||||
{"location": "message", "role": "system"},
|
||||
{"location": "tool_config"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def child_initial_input(
|
||||
*,
|
||||
name: str,
|
||||
|
||||
@@ -55,6 +55,35 @@ def test_child_initial_input_no_consecutive_same_role(parent_history: list[Any])
|
||||
assert all(prev != nxt for prev, nxt in pairwise(roles))
|
||||
|
||||
|
||||
def _cache_points(model_name: str) -> Any:
|
||||
extra = make_model_settings(None, model_name=model_name).extra_args or {}
|
||||
return extra.get("cache_control_injection_points")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[
|
||||
"bedrock/global.anthropic.claude-opus-4-8",
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
"openrouter/anthropic/claude-3.5-sonnet",
|
||||
],
|
||||
)
|
||||
def test_make_model_settings_enables_prompt_cache_for_claude(model_name: str) -> None:
|
||||
points = _cache_points(model_name)
|
||||
assert points == [
|
||||
{"location": "message", "role": "system"},
|
||||
{"location": "tool_config"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", ["gpt-5", "vertex_ai/gemini-2.5-pro", "openai/o3"])
|
||||
def test_make_model_settings_no_prompt_cache_for_non_claude(model_name: str) -> None:
|
||||
# No injection points for non-Claude models: the LiteLLM cache hook never
|
||||
# fires, so this stays a strict no-op (won't emit cache_control to strict
|
||||
# OpenAI-compatible endpoints).
|
||||
assert make_model_settings(None, model_name=model_name).extra_args is None
|
||||
|
||||
|
||||
def test_build_root_task_empty_config() -> None:
|
||||
assert build_root_task({}) == ""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user