From 2c5d4a166bc3176de931718c5c0fd877bd27dac8 Mon Sep 17 00:00:00 2001 From: Alex Schapiro Date: Mon, 10 Aug 2026 15:33:41 +0000 Subject: [PATCH] feat: serve denied turns on the fallback before pinning the agent --- docs/advanced/configuration.mdx | 4 ++-- strix/config/settings.py | 8 ++++---- strix/core/execution.py | 12 +++++++++--- tests/test_denied_retry_fallback.py | 24 +++++++++++++++++++++--- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/docs/advanced/configuration.mdx b/docs/advanced/configuration.mdx index a92b9003..63104bff 100644 --- a/docs/advanced/configuration.mdx +++ b/docs/advanced/configuration.mdx @@ -36,11 +36,11 @@ Configure Strix using environment variables or a config file. - Optional fallback model used after repeated content-guardrail denials within one agent's lifecycle. Unset disables this behavior. + Optional model that retries any turn blocked by a content guardrail. Unset disables this behavior, leaving a denial terminal for that agent. - Number of content-guardrail denials before the agent switches to `STRIX_LLM_FALLBACK` for the rest of its lifecycle. + Content-guardrail denials an agent may take before it is pinned to `STRIX_LLM_FALLBACK` for the rest of its lifecycle. Below that count it returns to the main model on the next turn. Counted per agent. diff --git a/strix/config/settings.py b/strix/config/settings.py index 190271b6..b33f0fa0 100644 --- a/strix/config/settings.py +++ b/strix/config/settings.py @@ -48,10 +48,10 @@ class LlmSettings(BaseSettings): default=False, alias="STRIX_FORCE_REQUIRED_TOOL_CHOICE", ) - # A model to fall back to for the rest of an agent's lifecycle once it has - # been content-denied ``denied_retries`` times (e.g. a ChatGPT-subscription - # cyber-risk guardrail block). Unset disables the fallback: a denial stays - # terminal for that agent as before. + # A model that serves any content-denied turn (e.g. a ChatGPT-subscription + # cyber-risk guardrail block), and that an agent is pinned to for the rest + # of its lifecycle once it has been denied ``denied_retries`` times. Unset + # disables the fallback: a denial stays terminal for that agent as before. fallback_model: str | None = Field(default=None, alias="STRIX_LLM_FALLBACK") denied_retries: int = Field(default=3, ge=0, alias="STRIX_LLM_DENIED_RETRIES") prompt_cache: bool = Field( diff --git a/strix/core/execution.py b/strix/core/execution.py index b2d03b93..67c4f9fa 100644 --- a/strix/core/execution.py +++ b/strix/core/execution.py @@ -644,9 +644,12 @@ async def _run_cycle( # noqa: PLR0912, PLR0915 image_strips = 0 compactions = 0 model_retries = 0 + retry_on_fallback = False while True: active_run_config = run_config - if coordinator.denial_fallback_model and await coordinator.is_on_denial_fallback(agent_id): + if coordinator.denial_fallback_model and ( + retry_on_fallback or await coordinator.is_on_denial_fallback(agent_id) + ): active_run_config = dataclasses.replace( run_config, model=coordinator.denial_fallback_model, @@ -654,6 +657,7 @@ async def _run_cycle( # noqa: PLR0912, PLR0915 coordinator.denial_fallback_model_settings or run_config.model_settings ), ) + retry_on_fallback = False stream: Any = None pre_run_items: list[Any] = [] try: @@ -775,10 +779,11 @@ async def _run_cycle( # noqa: PLR0912, PLR0915 and not await coordinator.is_on_denial_fallback(agent_id) ): denials = await coordinator.record_denial(agent_id) + retry_on_fallback = True if denials >= coordinator.denied_retries: await coordinator.mark_denial_fallback(agent_id) logger.warning( - "agent %s hit %d content denial(s); falling back to %s for the rest " + "agent %s hit %d content denial(s); pinned to %s for the rest " "of its lifecycle", agent_id, denials, @@ -786,10 +791,11 @@ async def _run_cycle( # noqa: PLR0912, PLR0915 ) else: logger.warning( - "agent %s content-denied (%d/%d); replaying the turn", + "agent %s content-denied (%d/%d); replaying this turn on %s", agent_id, denials, coordinator.denied_retries, + coordinator.denial_fallback_model, ) if session is not None: input_data = [] diff --git a/tests/test_denied_retry_fallback.py b/tests/test_denied_retry_fallback.py index 8b27f97c..e0079a50 100644 --- a/tests/test_denied_retry_fallback.py +++ b/tests/test_denied_retry_fallback.py @@ -75,7 +75,25 @@ async def _run_once( @pytest.mark.asyncio -async def test_run_cycle_falls_back_after_repeated_content_denials( +async def test_a_denied_turn_is_retried_on_the_fallback_model( + monkeypatch: pytest.MonkeyPatch, +) -> None: + streams = [_guardrail_stream(), _FakeStream()] + result, models, coordinator = await _run_once( + monkeypatch, + streams, + fallback_model="openai/gpt-5.4", + ) + + assert result is streams[1] + assert [model for model, _ in models] == ["openai/gpt-5.6-sol", "openai/gpt-5.4"] + # One denial is below the threshold, so the agent is not pinned to the + # fallback and its next turn starts on the main model again. + assert await coordinator.is_on_denial_fallback("root") is False + + +@pytest.mark.asyncio +async def test_agent_is_pinned_to_the_fallback_after_repeated_denials( monkeypatch: pytest.MonkeyPatch, ) -> None: streams = [_guardrail_stream() for _ in range(3)] + [_FakeStream()] @@ -86,7 +104,7 @@ async def test_run_cycle_falls_back_after_repeated_content_denials( ) assert result is streams[3] - assert [model for model, _ in models] == ["openai/gpt-5.6-sol"] * 3 + ["openai/gpt-5.4"] + assert [model for model, _ in models] == ["openai/gpt-5.6-sol"] + ["openai/gpt-5.4"] * 3 assert await coordinator.is_on_denial_fallback("root") is True @@ -100,7 +118,7 @@ async def test_run_cycle_does_not_retry_guardrail_without_fallback( @pytest.mark.asyncio -async def test_run_cycle_switches_on_first_denial_at_boundary( +async def test_run_cycle_pins_on_first_denial_at_boundary( monkeypatch: pytest.MonkeyPatch, ) -> None: streams = [_guardrail_stream(), _FakeStream()]