mirror of
https://github.com/usestrix/strix.git
synced 2026-08-18 17:52:32 +02:00
feat: fall back after repeated content denials
This commit is contained in:
@@ -35,6 +35,14 @@ Configure Strix using environment variables or a config file.
|
||||
Maximum number of retries for LLM API calls on transient failures.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="STRIX_LLM_FALLBACK" type="string">
|
||||
Optional fallback model used after repeated content-guardrail denials within one agent's lifecycle. Unset disables this behavior.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="STRIX_LLM_DENIED_RETRIES" default="3" type="integer">
|
||||
Number of content-guardrail denials before the agent switches to `STRIX_LLM_FALLBACK` for the rest of its lifecycle.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="STRIX_REASONING_EFFORT" default="high" type="string">
|
||||
Control thinking effort for reasoning models. Valid values: `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`. Defaults to `medium` for quick scan mode.
|
||||
</ParamField>
|
||||
|
||||
@@ -48,6 +48,12 @@ class LlmSettings(BaseSettings):
|
||||
default=False,
|
||||
alias="STRIX_FORCE_REQUIRED_TOOL_CHOICE",
|
||||
)
|
||||
# A model to fall back to for the rest of an agent's lifecycle once it has
|
||||
# been content-denied ``denied_retries`` times (e.g. a ChatGPT-subscription
|
||||
# cyber-risk guardrail block). Unset disables the fallback: a denial stays
|
||||
# terminal for that agent as before.
|
||||
fallback_model: str | None = Field(default=None, alias="STRIX_LLM_FALLBACK")
|
||||
denied_retries: int = Field(default=3, ge=0, alias="STRIX_LLM_DENIED_RETRIES")
|
||||
prompt_cache: bool = Field(
|
||||
default=True,
|
||||
alias="STRIX_PROMPT_CACHE",
|
||||
|
||||
@@ -53,6 +53,10 @@ class AgentCoordinator:
|
||||
self.errors: dict[str, str] = {}
|
||||
self.recovery_counts: dict[str, int] = {}
|
||||
self.idle_resume_counts: dict[str, int] = {}
|
||||
self.denial_counts: dict[str, int] = {}
|
||||
self.denial_fallback: set[str] = set()
|
||||
self._denial_fallback_model: str | None = None
|
||||
self._denied_retries: int = 3
|
||||
self.wait_kinds: dict[str, WaitKind] = {}
|
||||
self.runtimes: dict[str, AgentRuntime] = {}
|
||||
self._parent_notified: set[str] = set()
|
||||
@@ -67,6 +71,19 @@ class AgentCoordinator:
|
||||
def set_snapshot_path(self, path: Path) -> None:
|
||||
self._snapshot_path = path
|
||||
|
||||
def configure_denial_fallback(self, model: str | None, denied_retries: int) -> None:
|
||||
"""Configure the per-agent content denial fallback."""
|
||||
self._denial_fallback_model = model
|
||||
self._denied_retries = denied_retries
|
||||
|
||||
@property
|
||||
def denial_fallback_model(self) -> str | None:
|
||||
return self._denial_fallback_model
|
||||
|
||||
@property
|
||||
def denied_retries(self) -> int:
|
||||
return self._denied_retries
|
||||
|
||||
def mark_shutting_down(self) -> None:
|
||||
self.is_shutting_down = True
|
||||
|
||||
@@ -243,6 +260,27 @@ class AgentCoordinator:
|
||||
return
|
||||
await self._maybe_snapshot()
|
||||
|
||||
async def record_denial(self, agent_id: str) -> int:
|
||||
"""Count a content denial; return the new total."""
|
||||
async with self._lock:
|
||||
count = self.denial_counts.get(agent_id, 0) + 1
|
||||
self.denial_counts[agent_id] = count
|
||||
await self._maybe_snapshot()
|
||||
return count
|
||||
|
||||
async def mark_denial_fallback(self, agent_id: str) -> None:
|
||||
"""Mark an agent as using its content denial fallback."""
|
||||
async with self._lock:
|
||||
if agent_id in self.denial_fallback:
|
||||
return
|
||||
self.denial_fallback.add(agent_id)
|
||||
await self._maybe_snapshot()
|
||||
|
||||
async def is_on_denial_fallback(self, agent_id: str) -> bool:
|
||||
"""Return whether an agent is using its content denial fallback."""
|
||||
async with self._lock:
|
||||
return agent_id in self.denial_fallback
|
||||
|
||||
async def set_status(
|
||||
self, agent_id: str, status: Status | str, *, error: str | None = None
|
||||
) -> None:
|
||||
@@ -473,6 +511,8 @@ class AgentCoordinator:
|
||||
"pending_counts": dict(self.pending_counts),
|
||||
"recovery_counts": dict(self.recovery_counts),
|
||||
"idle_resume_counts": dict(self.idle_resume_counts),
|
||||
"denial_counts": dict(self.denial_counts),
|
||||
"denial_fallback": sorted(self.denial_fallback),
|
||||
"wait_kinds": dict(self.wait_kinds),
|
||||
"mailboxes": {
|
||||
aid: [dict(m) for m in runtime.mailbox]
|
||||
@@ -495,6 +535,8 @@ class AgentCoordinator:
|
||||
self.errors = dict(snap.get("errors", {}))
|
||||
self.recovery_counts = dict(snap.get("recovery_counts", {}))
|
||||
self.idle_resume_counts = dict(snap.get("idle_resume_counts", {}))
|
||||
self.denial_counts = dict(snap.get("denial_counts", {}))
|
||||
self.denial_fallback = set(snap.get("denial_fallback", []))
|
||||
self.wait_kinds = dict(snap.get("wait_kinds", {}))
|
||||
mailboxes = snap.get("mailboxes", {})
|
||||
if isinstance(mailboxes, dict):
|
||||
|
||||
+36
-3
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import dataclasses
|
||||
import logging
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
@@ -644,6 +645,11 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
compactions = 0
|
||||
model_retries = 0
|
||||
while True:
|
||||
active_run_config = run_config
|
||||
if coordinator.denial_fallback_model and await coordinator.is_on_denial_fallback(agent_id):
|
||||
active_run_config = dataclasses.replace(
|
||||
run_config, model=coordinator.denial_fallback_model
|
||||
)
|
||||
stream: Any = None
|
||||
pre_run_items: list[Any] = []
|
||||
try:
|
||||
@@ -656,7 +662,7 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
except Exception:
|
||||
logger.exception("image-budget enforcement failed for %s", agent_id)
|
||||
try:
|
||||
await _compact_session(agent, session, run_config, force=False)
|
||||
await _compact_session(agent, session, active_run_config, force=False)
|
||||
except Exception:
|
||||
logger.exception("proactive compaction failed for %s", agent_id)
|
||||
with contextlib.suppress(Exception):
|
||||
@@ -664,7 +670,7 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
stream = Runner.run_streamed(
|
||||
agent,
|
||||
input=input_data,
|
||||
run_config=run_config,
|
||||
run_config=active_run_config,
|
||||
context=context,
|
||||
max_turns=max_turns,
|
||||
session=session,
|
||||
@@ -744,7 +750,9 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
and is_context_overflow(exc)
|
||||
):
|
||||
try:
|
||||
compacted = await _compact_session(agent, session, run_config, force=True)
|
||||
compacted = await _compact_session(
|
||||
agent, session, active_run_config, force=True
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("overflow compaction recovery failed for %s", agent_id)
|
||||
compacted = False
|
||||
@@ -757,6 +765,31 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
)
|
||||
input_data = []
|
||||
continue
|
||||
if (
|
||||
coordinator.denial_fallback_model
|
||||
and codex.is_content_guardrail_error(exc)
|
||||
and not await coordinator.is_on_denial_fallback(agent_id)
|
||||
):
|
||||
denials = await coordinator.record_denial(agent_id)
|
||||
if denials >= coordinator.denied_retries:
|
||||
await coordinator.mark_denial_fallback(agent_id)
|
||||
logger.warning(
|
||||
"agent %s hit %d content denial(s); falling back to %s for the rest "
|
||||
"of its lifecycle",
|
||||
agent_id,
|
||||
denials,
|
||||
coordinator.denial_fallback_model,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"agent %s content-denied (%d/%d); replaying the turn",
|
||||
agent_id,
|
||||
denials,
|
||||
coordinator.denied_retries,
|
||||
)
|
||||
if session is not None:
|
||||
input_data = []
|
||||
continue
|
||||
if model_retries < _MAX_TRANSIENT_MODEL_RETRIES and _is_transient_model_error(exc):
|
||||
model_retries += 1
|
||||
delay = _transient_model_retry_delay(model_retries)
|
||||
|
||||
@@ -175,6 +175,10 @@ async def run_strix_scan(
|
||||
if coordinator is None:
|
||||
coordinator = AgentCoordinator()
|
||||
coordinator.set_snapshot_path(agents_path)
|
||||
coordinator.configure_denial_fallback(
|
||||
getattr(settings.llm, "fallback_model", None),
|
||||
getattr(settings.llm, "denied_retries", 3),
|
||||
)
|
||||
|
||||
from strix.tools.notes.tools import hydrate_notes_from_disk
|
||||
from strix.tools.todo.tools import hydrate_todos_from_disk
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from agents import RunConfig, Runner
|
||||
|
||||
from strix.config import codex
|
||||
from strix.core import execution
|
||||
from strix.core.agents import AgentCoordinator
|
||||
|
||||
|
||||
class _FakeStream:
|
||||
def __init__(self, exc: BaseException | None = None) -> None:
|
||||
self._exc = exc
|
||||
self.run_loop_exception: BaseException | None = None
|
||||
|
||||
async def stream_events(self) -> Any:
|
||||
if self._exc is not None:
|
||||
raise self._exc
|
||||
events: list[Any] = []
|
||||
for event in events:
|
||||
yield event
|
||||
|
||||
|
||||
def _patch_fast_backoff(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(execution, "_TRANSIENT_MODEL_RETRY_BASE_DELAY_S", 0.0)
|
||||
monkeypatch.setattr(execution, "_TRANSIENT_MODEL_RETRY_MAX_DELAY_S", 0.0)
|
||||
|
||||
|
||||
def _guardrail_stream() -> _FakeStream:
|
||||
return _FakeStream(codex.CodexContentGuardrailError("gpt-5.6-sol"))
|
||||
|
||||
|
||||
async def _run_once(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
streams: list[_FakeStream],
|
||||
*,
|
||||
fallback_model: str | None = None,
|
||||
denied_retries: int = 3,
|
||||
primary_model: str = "openai/gpt-5.6-sol",
|
||||
) -> tuple[Any, list[str | None], AgentCoordinator]:
|
||||
_patch_fast_backoff(monkeypatch)
|
||||
calls: list[str | None] = []
|
||||
|
||||
def _fake_run_streamed(*_args: Any, **kwargs: Any) -> _FakeStream:
|
||||
run_config = kwargs["run_config"]
|
||||
calls.append(run_config.model)
|
||||
return streams[len(calls) - 1]
|
||||
|
||||
monkeypatch.setattr(Runner, "run_streamed", _fake_run_streamed)
|
||||
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
if fallback_model is not None:
|
||||
coordinator.configure_denial_fallback(fallback_model, denied_retries)
|
||||
|
||||
result = await execution._run_cycle(
|
||||
object(),
|
||||
coordinator,
|
||||
"root",
|
||||
input_data="task",
|
||||
run_config=RunConfig(model=primary_model),
|
||||
context={},
|
||||
max_turns=5,
|
||||
session=None,
|
||||
interactive=False,
|
||||
event_sink=None,
|
||||
hooks=None,
|
||||
)
|
||||
return result, calls, coordinator
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cycle_falls_back_after_repeated_content_denials(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
streams = [_guardrail_stream() for _ in range(3)] + [_FakeStream()]
|
||||
result, models, coordinator = await _run_once(
|
||||
monkeypatch,
|
||||
streams,
|
||||
fallback_model="openai/gpt-5.4",
|
||||
)
|
||||
|
||||
assert result is streams[3]
|
||||
assert models == ["openai/gpt-5.6-sol"] * 3 + ["openai/gpt-5.4"]
|
||||
assert await coordinator.is_on_denial_fallback("root") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cycle_does_not_retry_guardrail_without_fallback(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
guardrail = codex.CodexContentGuardrailError("gpt-5.6-sol")
|
||||
with pytest.raises(codex.CodexContentGuardrailError):
|
||||
await _run_once(monkeypatch, [_FakeStream(guardrail), _FakeStream()])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cycle_switches_on_first_denial_at_boundary(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
streams = [_guardrail_stream(), _FakeStream()]
|
||||
result, models, coordinator = await _run_once(
|
||||
monkeypatch,
|
||||
streams,
|
||||
fallback_model="openai/gpt-5.4",
|
||||
denied_retries=1,
|
||||
)
|
||||
|
||||
assert result is streams[1]
|
||||
assert models == ["openai/gpt-5.6-sol", "openai/gpt-5.4"]
|
||||
assert await coordinator.is_on_denial_fallback("root") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_denial_fallback_state_round_trips_through_snapshot() -> None:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
coordinator.configure_denial_fallback("openai/gpt-5.4", 3)
|
||||
await coordinator.record_denial("root")
|
||||
await coordinator.mark_denial_fallback("root")
|
||||
|
||||
restored = AgentCoordinator()
|
||||
await restored.restore(await coordinator.snapshot())
|
||||
|
||||
assert restored.denial_counts == {"root": 1}
|
||||
assert await restored.is_on_denial_fallback("root") is True
|
||||
Reference in New Issue
Block a user