feat: serve denied turns on the fallback before pinning the agent

This commit is contained in:
Alex Schapiro
2026-08-10 15:33:41 +00:00
parent 009cc94220
commit 2c5d4a166b
4 changed files with 36 additions and 12 deletions
+2 -2
View File
@@ -36,11 +36,11 @@ Configure Strix using environment variables or a config file.
</ParamField>
<ParamField path="STRIX_LLM_FALLBACK" type="string">
Optional fallback model used after repeated content-guardrail denials within one agent's lifecycle. Unset disables this behavior.
Optional model that retries any turn blocked by a content guardrail. Unset disables this behavior, leaving a denial terminal for that agent.
</ParamField>
<ParamField path="STRIX_LLM_DENIED_RETRIES" default="3" type="integer">
Number of content-guardrail denials before the agent switches to `STRIX_LLM_FALLBACK` for the rest of its lifecycle.
Content-guardrail denials an agent may take before it is pinned to `STRIX_LLM_FALLBACK` for the rest of its lifecycle. Below that count it returns to the main model on the next turn. Counted per agent.
</ParamField>
<ParamField path="STRIX_REASONING_EFFORT" default="high" type="string">
+4 -4
View File
@@ -48,10 +48,10 @@ class LlmSettings(BaseSettings):
default=False,
alias="STRIX_FORCE_REQUIRED_TOOL_CHOICE",
)
# A model to fall back to for the rest of an agent's lifecycle once it has
# been content-denied ``denied_retries`` times (e.g. a ChatGPT-subscription
# cyber-risk guardrail block). Unset disables the fallback: a denial stays
# terminal for that agent as before.
# A model that serves any content-denied turn (e.g. a ChatGPT-subscription
# cyber-risk guardrail block), and that an agent is pinned to for the rest
# of its lifecycle once it has been denied ``denied_retries`` times. Unset
# disables the fallback: a denial stays terminal for that agent as before.
fallback_model: str | None = Field(default=None, alias="STRIX_LLM_FALLBACK")
denied_retries: int = Field(default=3, ge=0, alias="STRIX_LLM_DENIED_RETRIES")
prompt_cache: bool = Field(
+9 -3
View File
@@ -644,9 +644,12 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
image_strips = 0
compactions = 0
model_retries = 0
retry_on_fallback = False
while True:
active_run_config = run_config
if coordinator.denial_fallback_model and await coordinator.is_on_denial_fallback(agent_id):
if coordinator.denial_fallback_model and (
retry_on_fallback or await coordinator.is_on_denial_fallback(agent_id)
):
active_run_config = dataclasses.replace(
run_config,
model=coordinator.denial_fallback_model,
@@ -654,6 +657,7 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
coordinator.denial_fallback_model_settings or run_config.model_settings
),
)
retry_on_fallback = False
stream: Any = None
pre_run_items: list[Any] = []
try:
@@ -775,10 +779,11 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
and not await coordinator.is_on_denial_fallback(agent_id)
):
denials = await coordinator.record_denial(agent_id)
retry_on_fallback = True
if denials >= coordinator.denied_retries:
await coordinator.mark_denial_fallback(agent_id)
logger.warning(
"agent %s hit %d content denial(s); falling back to %s for the rest "
"agent %s hit %d content denial(s); pinned to %s for the rest "
"of its lifecycle",
agent_id,
denials,
@@ -786,10 +791,11 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
)
else:
logger.warning(
"agent %s content-denied (%d/%d); replaying the turn",
"agent %s content-denied (%d/%d); replaying this turn on %s",
agent_id,
denials,
coordinator.denied_retries,
coordinator.denial_fallback_model,
)
if session is not None:
input_data = []
+21 -3
View File
@@ -75,7 +75,25 @@ async def _run_once(
@pytest.mark.asyncio
async def test_run_cycle_falls_back_after_repeated_content_denials(
async def test_a_denied_turn_is_retried_on_the_fallback_model(
monkeypatch: pytest.MonkeyPatch,
) -> None:
streams = [_guardrail_stream(), _FakeStream()]
result, models, coordinator = await _run_once(
monkeypatch,
streams,
fallback_model="openai/gpt-5.4",
)
assert result is streams[1]
assert [model for model, _ in models] == ["openai/gpt-5.6-sol", "openai/gpt-5.4"]
# One denial is below the threshold, so the agent is not pinned to the
# fallback and its next turn starts on the main model again.
assert await coordinator.is_on_denial_fallback("root") is False
@pytest.mark.asyncio
async def test_agent_is_pinned_to_the_fallback_after_repeated_denials(
monkeypatch: pytest.MonkeyPatch,
) -> None:
streams = [_guardrail_stream() for _ in range(3)] + [_FakeStream()]
@@ -86,7 +104,7 @@ async def test_run_cycle_falls_back_after_repeated_content_denials(
)
assert result is streams[3]
assert [model for model, _ in models] == ["openai/gpt-5.6-sol"] * 3 + ["openai/gpt-5.4"]
assert [model for model, _ in models] == ["openai/gpt-5.6-sol"] + ["openai/gpt-5.4"] * 3
assert await coordinator.is_on_denial_fallback("root") is True
@@ -100,7 +118,7 @@ async def test_run_cycle_does_not_retry_guardrail_without_fallback(
@pytest.mark.asyncio
async def test_run_cycle_switches_on_first_denial_at_boundary(
async def test_run_cycle_pins_on_first_denial_at_boundary(
monkeypatch: pytest.MonkeyPatch,
) -> None:
streams = [_guardrail_stream(), _FakeStream()]