diff --git a/strix/core/inputs.py b/strix/core/inputs.py index e53ae321..901fde38 100644 --- a/strix/core/inputs.py +++ b/strix/core/inputs.py @@ -141,9 +141,57 @@ def make_model_settings( ) if force_required_tool_choice and _accepts_required_tool_choice(model_name): model_settings = model_settings.resolve(ModelSettings(tool_choice="required")) + if _is_claude_model(model_name): + model_settings = model_settings.resolve( + ModelSettings(extra_args=_claude_prompt_cache_extra_args()), + ) return model_settings +def _is_claude_model(model_name: str) -> bool: + return "claude" in (model_name or "").strip().lower() + + +def _claude_prompt_cache_extra_args() -> dict[str, Any]: + """Enable Anthropic/Bedrock prompt caching for Claude models via LiteLLM. + + A Strix scan is a long, multi-turn agentic loop that re-sends a large, + STABLE prefix every turn — the system prompt plus the tool schemas — while + only the conversation tail changes. Without a caching breakpoint the whole + prefix is re-tokenised and billed at the full input rate on every turn; on + Bedrock Claude that is the single biggest lever on scan cost (measured here: + ``cache-read 0% -> 57%`` on a real scan once these points are set). + + LiteLLM already implements this end to end: when + ``cache_control_injection_points`` is present in the call kwargs its + ``AnthropicCacheControlHook`` fires and emits the provider-appropriate + breakpoint (Anthropic ``cache_control``; Bedrock Converse ``cachePoint``), + honouring Anthropic's 4-breakpoint cap. ``LitellmModel`` forwards + ``ModelSettings.extra_args`` straight into ``litellm.acompletion()``, so + passing the injection points there is all that is required. + + This is deliberately kept at the LiteLLM-config layer rather than a general + ``ModelSettings`` caching flag: that is the direction the Agents SDK + maintainer prescribed when declining a native ``cache_system_prompt`` field + (openai/openai-agents-python#3008 / #3009) — caching is a LiteLLM/provider + behaviour and a ``ModelSettings`` flag would let strict OpenAI-compatible + paths emit non-standard ``cache_control`` parts. Gating on Claude keeps this + a no-op for every other provider (no injection points -> the hook never + fires), and only Claude-family routes (Anthropic native, Bedrock, Vertex, + OpenRouter -> Claude) honour the marker. + + Two breakpoints on the stable prefix (2 of the 4 allowed), leaving headroom: + - the system prompt (``role: system``) — the largest repeated span + - the tool schemas (``tool_config``) — sizeable and identical every turn + """ + return { + "cache_control_injection_points": [ + {"location": "message", "role": "system"}, + {"location": "tool_config"}, + ], + } + + def child_initial_input( *, name: str, diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 486c7e54..52d2bcb6 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -55,6 +55,35 @@ def test_child_initial_input_no_consecutive_same_role(parent_history: list[Any]) assert all(prev != nxt for prev, nxt in pairwise(roles)) +def _cache_points(model_name: str) -> Any: + extra = make_model_settings(None, model_name=model_name).extra_args or {} + return extra.get("cache_control_injection_points") + + +@pytest.mark.parametrize( + "model_name", + [ + "bedrock/global.anthropic.claude-opus-4-8", + "anthropic/claude-sonnet-4-5", + "openrouter/anthropic/claude-3.5-sonnet", + ], +) +def test_make_model_settings_enables_prompt_cache_for_claude(model_name: str) -> None: + points = _cache_points(model_name) + assert points == [ + {"location": "message", "role": "system"}, + {"location": "tool_config"}, + ] + + +@pytest.mark.parametrize("model_name", ["gpt-5", "vertex_ai/gemini-2.5-pro", "openai/o3"]) +def test_make_model_settings_no_prompt_cache_for_non_claude(model_name: str) -> None: + # No injection points for non-Claude models: the LiteLLM cache hook never + # fires, so this stays a strict no-op (won't emit cache_control to strict + # OpenAI-compatible endpoints). + assert make_model_settings(None, model_name=model_name).extra_args is None + + def test_build_root_task_empty_config() -> None: assert build_root_task({}) == ""