mirror of
https://github.com/usestrix/strix.git
synced 2026-08-16 09:26:39 +02:00
feat: serve denied turns on the fallback before pinning the agent
This commit is contained in:
@@ -36,11 +36,11 @@ Configure Strix using environment variables or a config file.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="STRIX_LLM_FALLBACK" type="string">
|
||||
Optional fallback model used after repeated content-guardrail denials within one agent's lifecycle. Unset disables this behavior.
|
||||
Optional model that retries any turn blocked by a content guardrail. Unset disables this behavior, leaving a denial terminal for that agent.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="STRIX_LLM_DENIED_RETRIES" default="3" type="integer">
|
||||
Number of content-guardrail denials before the agent switches to `STRIX_LLM_FALLBACK` for the rest of its lifecycle.
|
||||
Content-guardrail denials an agent may take before it is pinned to `STRIX_LLM_FALLBACK` for the rest of its lifecycle. Below that count it returns to the main model on the next turn. Counted per agent.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="STRIX_REASONING_EFFORT" default="high" type="string">
|
||||
|
||||
@@ -48,10 +48,10 @@ class LlmSettings(BaseSettings):
|
||||
default=False,
|
||||
alias="STRIX_FORCE_REQUIRED_TOOL_CHOICE",
|
||||
)
|
||||
# A model to fall back to for the rest of an agent's lifecycle once it has
|
||||
# been content-denied ``denied_retries`` times (e.g. a ChatGPT-subscription
|
||||
# cyber-risk guardrail block). Unset disables the fallback: a denial stays
|
||||
# terminal for that agent as before.
|
||||
# A model that serves any content-denied turn (e.g. a ChatGPT-subscription
|
||||
# cyber-risk guardrail block), and that an agent is pinned to for the rest
|
||||
# of its lifecycle once it has been denied ``denied_retries`` times. Unset
|
||||
# disables the fallback: a denial stays terminal for that agent as before.
|
||||
fallback_model: str | None = Field(default=None, alias="STRIX_LLM_FALLBACK")
|
||||
denied_retries: int = Field(default=3, ge=0, alias="STRIX_LLM_DENIED_RETRIES")
|
||||
prompt_cache: bool = Field(
|
||||
|
||||
@@ -644,9 +644,12 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
image_strips = 0
|
||||
compactions = 0
|
||||
model_retries = 0
|
||||
retry_on_fallback = False
|
||||
while True:
|
||||
active_run_config = run_config
|
||||
if coordinator.denial_fallback_model and await coordinator.is_on_denial_fallback(agent_id):
|
||||
if coordinator.denial_fallback_model and (
|
||||
retry_on_fallback or await coordinator.is_on_denial_fallback(agent_id)
|
||||
):
|
||||
active_run_config = dataclasses.replace(
|
||||
run_config,
|
||||
model=coordinator.denial_fallback_model,
|
||||
@@ -654,6 +657,7 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
coordinator.denial_fallback_model_settings or run_config.model_settings
|
||||
),
|
||||
)
|
||||
retry_on_fallback = False
|
||||
stream: Any = None
|
||||
pre_run_items: list[Any] = []
|
||||
try:
|
||||
@@ -775,10 +779,11 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
and not await coordinator.is_on_denial_fallback(agent_id)
|
||||
):
|
||||
denials = await coordinator.record_denial(agent_id)
|
||||
retry_on_fallback = True
|
||||
if denials >= coordinator.denied_retries:
|
||||
await coordinator.mark_denial_fallback(agent_id)
|
||||
logger.warning(
|
||||
"agent %s hit %d content denial(s); falling back to %s for the rest "
|
||||
"agent %s hit %d content denial(s); pinned to %s for the rest "
|
||||
"of its lifecycle",
|
||||
agent_id,
|
||||
denials,
|
||||
@@ -786,10 +791,11 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"agent %s content-denied (%d/%d); replaying the turn",
|
||||
"agent %s content-denied (%d/%d); replaying this turn on %s",
|
||||
agent_id,
|
||||
denials,
|
||||
coordinator.denied_retries,
|
||||
coordinator.denial_fallback_model,
|
||||
)
|
||||
if session is not None:
|
||||
input_data = []
|
||||
|
||||
@@ -75,7 +75,25 @@ async def _run_once(
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cycle_falls_back_after_repeated_content_denials(
|
||||
async def test_a_denied_turn_is_retried_on_the_fallback_model(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
streams = [_guardrail_stream(), _FakeStream()]
|
||||
result, models, coordinator = await _run_once(
|
||||
monkeypatch,
|
||||
streams,
|
||||
fallback_model="openai/gpt-5.4",
|
||||
)
|
||||
|
||||
assert result is streams[1]
|
||||
assert [model for model, _ in models] == ["openai/gpt-5.6-sol", "openai/gpt-5.4"]
|
||||
# One denial is below the threshold, so the agent is not pinned to the
|
||||
# fallback and its next turn starts on the main model again.
|
||||
assert await coordinator.is_on_denial_fallback("root") is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_is_pinned_to_the_fallback_after_repeated_denials(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
streams = [_guardrail_stream() for _ in range(3)] + [_FakeStream()]
|
||||
@@ -86,7 +104,7 @@ async def test_run_cycle_falls_back_after_repeated_content_denials(
|
||||
)
|
||||
|
||||
assert result is streams[3]
|
||||
assert [model for model, _ in models] == ["openai/gpt-5.6-sol"] * 3 + ["openai/gpt-5.4"]
|
||||
assert [model for model, _ in models] == ["openai/gpt-5.6-sol"] + ["openai/gpt-5.4"] * 3
|
||||
assert await coordinator.is_on_denial_fallback("root") is True
|
||||
|
||||
|
||||
@@ -100,7 +118,7 @@ async def test_run_cycle_does_not_retry_guardrail_without_fallback(
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cycle_switches_on_first_denial_at_boundary(
|
||||
async def test_run_cycle_pins_on_first_denial_at_boundary(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
streams = [_guardrail_stream(), _FakeStream()]
|
||||
|
||||
Reference in New Issue
Block a user