Compare commits

...
11 changed files with 332 additions and 3 deletions
+8
View File
@@ -35,6 +35,14 @@ Configure Strix using environment variables or a config file.
Maximum number of retries for LLM API calls on transient failures. Maximum number of retries for LLM API calls on transient failures.
</ParamField> </ParamField>
<ParamField path="STRIX_LLM_FALLBACK" type="string">
Optional model that retries any turn blocked by a content guardrail. Unset disables this behavior, leaving a denial terminal for that agent.
</ParamField>
<ParamField path="STRIX_LLM_DENIED_RETRIES" default="3" type="integer">
Content-guardrail denials an agent may take before it is pinned to `STRIX_LLM_FALLBACK` for the rest of its lifecycle. Below that count it returns to the main model on the next turn. Counted per agent.
</ParamField>
<ParamField path="STRIX_REASONING_EFFORT" default="high" type="string"> <ParamField path="STRIX_REASONING_EFFORT" default="high" type="string">
Control thinking effort for reasoning models. Valid values: `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`. Defaults to `medium` for quick scan mode. Control thinking effort for reasoning models. Valid values: `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`. Defaults to `medium` for quick scan mode.
</ParamField> </ParamField>
+19
View File
@@ -733,6 +733,25 @@ def _configure_litellm_default(name: str, value: str) -> None:
setattr(litellm, name, value) setattr(litellm, name, value)
def fallback_model_rejection(fallback: str, primary: str, settings: Settings) -> str | None:
"""Why ``fallback`` cannot stand in for ``primary`` mid-run, or None if it can.
Provider credentials, SDK route, and each agent's tool wrappers are all set
up once from the primary model, so a fallback that needs a different
provider or tool schema would be rejected on every request it serves.
"""
if (
_split_model_provider(_normalized_model_name(fallback))[0]
!= (_split_model_provider(_normalized_model_name(primary))[0])
):
return "needs a different provider, whose credentials are not configured"
if uses_chat_completions_tool_schema(fallback, settings) != uses_chat_completions_tool_schema(
primary, settings
):
return "needs a different tool schema than the agents are built with"
return None
def uses_chat_completions_tool_schema(model_name: str, settings: Settings) -> bool: def uses_chat_completions_tool_schema(model_name: str, settings: Settings) -> bool:
"""Return whether the resolved SDK route can only receive JSON function tools.""" """Return whether the resolved SDK route can only receive JSON function tools."""
if codex.subscription_model(model_name): if codex.subscription_model(model_name):
+6
View File
@@ -48,6 +48,12 @@ class LlmSettings(BaseSettings):
default=False, default=False,
alias="STRIX_FORCE_REQUIRED_TOOL_CHOICE", alias="STRIX_FORCE_REQUIRED_TOOL_CHOICE",
) )
# A model that serves any content-denied turn (e.g. a ChatGPT-subscription
# cyber-risk guardrail block), and that an agent is pinned to for the rest
# of its lifecycle once it has been denied ``denied_retries`` times. Unset
# disables the fallback: a denial stays terminal for that agent as before.
fallback_model: str | None = Field(default=None, alias="STRIX_LLM_FALLBACK")
denied_retries: int = Field(default=3, ge=0, alias="STRIX_LLM_DENIED_RETRIES")
prompt_cache: bool = Field( prompt_cache: bool = Field(
default=True, default=True,
alias="STRIX_PROMPT_CACHE", alias="STRIX_PROMPT_CACHE",
+54
View File
@@ -18,6 +18,7 @@ if TYPE_CHECKING:
from agents.items import TResponseInputItem from agents.items import TResponseInputItem
from agents.memory import Session from agents.memory import Session
from agents.model_settings import ModelSettings
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -53,6 +54,11 @@ class AgentCoordinator:
self.errors: dict[str, str] = {} self.errors: dict[str, str] = {}
self.recovery_counts: dict[str, int] = {} self.recovery_counts: dict[str, int] = {}
self.idle_resume_counts: dict[str, int] = {} self.idle_resume_counts: dict[str, int] = {}
self.denial_counts: dict[str, int] = {}
self.denial_fallback: set[str] = set()
self._denial_fallback_model: str | None = None
self._denial_fallback_model_settings: ModelSettings | None = None
self._denied_retries: int = 3
self.wait_kinds: dict[str, WaitKind] = {} self.wait_kinds: dict[str, WaitKind] = {}
self.runtimes: dict[str, AgentRuntime] = {} self.runtimes: dict[str, AgentRuntime] = {}
self._parent_notified: set[str] = set() self._parent_notified: set[str] = set()
@@ -67,6 +73,29 @@ class AgentCoordinator:
def set_snapshot_path(self, path: Path) -> None: def set_snapshot_path(self, path: Path) -> None:
self._snapshot_path = path self._snapshot_path = path
def configure_denial_fallback(
self,
model: str | None,
denied_retries: int,
model_settings: ModelSettings | None = None,
) -> None:
"""Configure the per-agent content denial fallback."""
self._denial_fallback_model = model
self._denial_fallback_model_settings = model_settings
self._denied_retries = denied_retries
@property
def denial_fallback_model(self) -> str | None:
return self._denial_fallback_model
@property
def denial_fallback_model_settings(self) -> ModelSettings | None:
return self._denial_fallback_model_settings
@property
def denied_retries(self) -> int:
return self._denied_retries
def mark_shutting_down(self) -> None: def mark_shutting_down(self) -> None:
self.is_shutting_down = True self.is_shutting_down = True
@@ -243,6 +272,27 @@ class AgentCoordinator:
return return
await self._maybe_snapshot() await self._maybe_snapshot()
async def record_denial(self, agent_id: str) -> int:
"""Count a content denial; return the new total."""
async with self._lock:
count = self.denial_counts.get(agent_id, 0) + 1
self.denial_counts[agent_id] = count
await self._maybe_snapshot()
return count
async def mark_denial_fallback(self, agent_id: str) -> None:
"""Mark an agent as using its content denial fallback."""
async with self._lock:
if agent_id in self.denial_fallback:
return
self.denial_fallback.add(agent_id)
await self._maybe_snapshot()
async def is_on_denial_fallback(self, agent_id: str) -> bool:
"""Return whether an agent is using its content denial fallback."""
async with self._lock:
return agent_id in self.denial_fallback
async def set_status( async def set_status(
self, agent_id: str, status: Status | str, *, error: str | None = None self, agent_id: str, status: Status | str, *, error: str | None = None
) -> None: ) -> None:
@@ -473,6 +523,8 @@ class AgentCoordinator:
"pending_counts": dict(self.pending_counts), "pending_counts": dict(self.pending_counts),
"recovery_counts": dict(self.recovery_counts), "recovery_counts": dict(self.recovery_counts),
"idle_resume_counts": dict(self.idle_resume_counts), "idle_resume_counts": dict(self.idle_resume_counts),
"denial_counts": dict(self.denial_counts),
"denial_fallback": sorted(self.denial_fallback),
"wait_kinds": dict(self.wait_kinds), "wait_kinds": dict(self.wait_kinds),
"mailboxes": { "mailboxes": {
aid: [dict(m) for m in runtime.mailbox] aid: [dict(m) for m in runtime.mailbox]
@@ -495,6 +547,8 @@ class AgentCoordinator:
self.errors = dict(snap.get("errors", {})) self.errors = dict(snap.get("errors", {}))
self.recovery_counts = dict(snap.get("recovery_counts", {})) self.recovery_counts = dict(snap.get("recovery_counts", {}))
self.idle_resume_counts = dict(snap.get("idle_resume_counts", {})) self.idle_resume_counts = dict(snap.get("idle_resume_counts", {}))
self.denial_counts = dict(snap.get("denial_counts", {}))
self.denial_fallback = set(snap.get("denial_fallback", []))
self.wait_kinds = dict(snap.get("wait_kinds", {})) self.wait_kinds = dict(snap.get("wait_kinds", {}))
mailboxes = snap.get("mailboxes", {}) mailboxes = snap.get("mailboxes", {})
if isinstance(mailboxes, dict): if isinstance(mailboxes, dict):
+46 -3
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio import asyncio
import contextlib import contextlib
import dataclasses
import logging import logging
import uuid import uuid
from collections.abc import Callable from collections.abc import Callable
@@ -643,7 +644,20 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
image_strips = 0 image_strips = 0
compactions = 0 compactions = 0
model_retries = 0 model_retries = 0
retry_on_fallback = False
while True: while True:
active_run_config = run_config
if coordinator.denial_fallback_model and (
retry_on_fallback or await coordinator.is_on_denial_fallback(agent_id)
):
active_run_config = dataclasses.replace(
run_config,
model=coordinator.denial_fallback_model,
model_settings=(
coordinator.denial_fallback_model_settings or run_config.model_settings
),
)
retry_on_fallback = False
stream: Any = None stream: Any = None
pre_run_items: list[Any] = [] pre_run_items: list[Any] = []
try: try:
@@ -656,7 +670,7 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
except Exception: except Exception:
logger.exception("image-budget enforcement failed for %s", agent_id) logger.exception("image-budget enforcement failed for %s", agent_id)
try: try:
await _compact_session(agent, session, run_config, force=False) await _compact_session(agent, session, active_run_config, force=False)
except Exception: except Exception:
logger.exception("proactive compaction failed for %s", agent_id) logger.exception("proactive compaction failed for %s", agent_id)
with contextlib.suppress(Exception): with contextlib.suppress(Exception):
@@ -664,7 +678,7 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
stream = Runner.run_streamed( stream = Runner.run_streamed(
agent, agent,
input=input_data, input=input_data,
run_config=run_config, run_config=active_run_config,
context=context, context=context,
max_turns=max_turns, max_turns=max_turns,
session=session, session=session,
@@ -744,7 +758,9 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
and is_context_overflow(exc) and is_context_overflow(exc)
): ):
try: try:
compacted = await _compact_session(agent, session, run_config, force=True) compacted = await _compact_session(
agent, session, active_run_config, force=True
)
except Exception: except Exception:
logger.exception("overflow compaction recovery failed for %s", agent_id) logger.exception("overflow compaction recovery failed for %s", agent_id)
compacted = False compacted = False
@@ -757,6 +773,33 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
) )
input_data = [] input_data = []
continue continue
if (
coordinator.denial_fallback_model
and codex.is_content_guardrail_error(exc)
and not await coordinator.is_on_denial_fallback(agent_id)
):
denials = await coordinator.record_denial(agent_id)
retry_on_fallback = True
if denials >= coordinator.denied_retries:
await coordinator.mark_denial_fallback(agent_id)
logger.warning(
"agent %s hit %d content denial(s); pinned to %s for the rest "
"of its lifecycle",
agent_id,
denials,
coordinator.denial_fallback_model,
)
else:
logger.warning(
"agent %s content-denied (%d/%d); replaying this turn on %s",
agent_id,
denials,
coordinator.denied_retries,
coordinator.denial_fallback_model,
)
if session is not None:
input_data = []
continue
if model_retries < _MAX_TRANSIENT_MODEL_RETRIES and _is_transient_model_error(exc): if model_retries < _MAX_TRANSIENT_MODEL_RETRIES and _is_transient_model_error(exc):
model_retries += 1 model_retries += 1
delay = _transient_model_retry_delay(model_retries) delay = _transient_model_retry_delay(model_retries)
+23
View File
@@ -22,6 +22,7 @@ from strix.config import load_settings
from strix.config.models import ( from strix.config.models import (
StrixProvider, StrixProvider,
configure_sdk_model_defaults, configure_sdk_model_defaults,
fallback_model_rejection,
uses_chat_completions_tool_schema, uses_chat_completions_tool_schema,
) )
from strix.config.settings import DEFAULT_MAX_TURNS from strix.config.settings import DEFAULT_MAX_TURNS
@@ -175,6 +176,28 @@ async def run_strix_scan(
if coordinator is None: if coordinator is None:
coordinator = AgentCoordinator() coordinator = AgentCoordinator()
coordinator.set_snapshot_path(agents_path) coordinator.set_snapshot_path(agents_path)
fallback_model = settings.llm.fallback_model
if fallback_model and (
rejection := fallback_model_rejection(fallback_model, resolved_model, settings)
):
raise RuntimeError(
f"STRIX_LLM_FALLBACK '{fallback_model}' {rejection}; it could never serve a "
f"turn for '{resolved_model}'. Pick a fallback from the same provider family."
)
coordinator.configure_denial_fallback(
fallback_model,
settings.llm.denied_retries,
model_settings=make_model_settings(
settings.llm.reasoning_effort,
model_name=fallback_model,
force_required_tool_choice=settings.llm.force_required_tool_choice,
request_timeout=settings.llm.timeout,
prompt_cache=settings.llm.prompt_cache,
extra_headers=settings.llm.extra_headers,
)
if fallback_model
else None,
)
from strix.tools.notes.tools import hydrate_notes_from_disk from strix.tools.notes.tools import hydrate_notes_from_disk
from strix.tools.todo.tools import hydrate_todos_from_disk from strix.tools.todo.tools import hydrate_todos_from_disk
+168
View File
@@ -0,0 +1,168 @@
from __future__ import annotations
from typing import Any
import pytest
from agents import ModelSettings, RunConfig, Runner
from strix.config import codex
from strix.core import execution
from strix.core.agents import AgentCoordinator
class _FakeStream:
def __init__(self, exc: BaseException | None = None) -> None:
self._exc = exc
self.run_loop_exception: BaseException | None = None
async def stream_events(self) -> Any:
if self._exc is not None:
raise self._exc
events: list[Any] = []
for event in events:
yield event
def _patch_fast_backoff(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(execution, "_TRANSIENT_MODEL_RETRY_BASE_DELAY_S", 0.0)
monkeypatch.setattr(execution, "_TRANSIENT_MODEL_RETRY_MAX_DELAY_S", 0.0)
def _guardrail_stream() -> _FakeStream:
return _FakeStream(codex.CodexContentGuardrailError("gpt-5.6-sol"))
async def _run_once(
monkeypatch: pytest.MonkeyPatch,
streams: list[_FakeStream],
*,
fallback_model: str | None = None,
denied_retries: int = 3,
primary_model: str = "openai/gpt-5.6-sol",
fallback_model_settings: ModelSettings | None = None,
) -> tuple[Any, list[tuple[str | None, ModelSettings]], AgentCoordinator]:
_patch_fast_backoff(monkeypatch)
calls: list[tuple[str | None, ModelSettings]] = []
def _fake_run_streamed(*_args: Any, **kwargs: Any) -> _FakeStream:
run_config = kwargs["run_config"]
calls.append((run_config.model, run_config.model_settings))
return streams[len(calls) - 1]
monkeypatch.setattr(Runner, "run_streamed", _fake_run_streamed)
coordinator = AgentCoordinator()
await coordinator.register("root", "strix", parent_id=None)
if fallback_model is not None:
coordinator.configure_denial_fallback(
fallback_model, denied_retries, model_settings=fallback_model_settings
)
result = await execution._run_cycle(
object(),
coordinator,
"root",
input_data="task",
run_config=RunConfig(model=primary_model, model_settings=ModelSettings()),
context={},
max_turns=5,
session=None,
interactive=False,
event_sink=None,
hooks=None,
)
return result, calls, coordinator
@pytest.mark.asyncio
async def test_a_denied_turn_is_retried_on_the_fallback_model(
monkeypatch: pytest.MonkeyPatch,
) -> None:
streams = [_guardrail_stream(), _FakeStream()]
result, models, coordinator = await _run_once(
monkeypatch,
streams,
fallback_model="openai/gpt-5.4",
)
assert result is streams[1]
assert [model for model, _ in models] == ["openai/gpt-5.6-sol", "openai/gpt-5.4"]
# One denial is below the threshold, so the agent is not pinned to the
# fallback and its next turn starts on the main model again.
assert await coordinator.is_on_denial_fallback("root") is False
@pytest.mark.asyncio
async def test_agent_is_pinned_to_the_fallback_after_repeated_denials(
monkeypatch: pytest.MonkeyPatch,
) -> None:
streams = [_guardrail_stream() for _ in range(3)] + [_FakeStream()]
result, models, coordinator = await _run_once(
monkeypatch,
streams,
fallback_model="openai/gpt-5.4",
)
assert result is streams[3]
assert [model for model, _ in models] == ["openai/gpt-5.6-sol"] + ["openai/gpt-5.4"] * 3
assert await coordinator.is_on_denial_fallback("root") is True
@pytest.mark.asyncio
async def test_run_cycle_does_not_retry_guardrail_without_fallback(
monkeypatch: pytest.MonkeyPatch,
) -> None:
guardrail = codex.CodexContentGuardrailError("gpt-5.6-sol")
with pytest.raises(codex.CodexContentGuardrailError):
await _run_once(monkeypatch, [_FakeStream(guardrail), _FakeStream()])
@pytest.mark.asyncio
async def test_run_cycle_pins_on_first_denial_at_boundary(
monkeypatch: pytest.MonkeyPatch,
) -> None:
streams = [_guardrail_stream(), _FakeStream()]
result, models, coordinator = await _run_once(
monkeypatch,
streams,
fallback_model="openai/gpt-5.4",
denied_retries=1,
)
assert result is streams[1]
assert [model for model, _ in models] == ["openai/gpt-5.6-sol", "openai/gpt-5.4"]
assert await coordinator.is_on_denial_fallback("root") is True
@pytest.mark.asyncio
async def test_fallback_uses_its_own_model_settings(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fallback_settings = ModelSettings(parallel_tool_calls=True)
streams = [_guardrail_stream(), _FakeStream()]
result, calls, _coordinator = await _run_once(
monkeypatch,
streams,
fallback_model="openai/gpt-5.4",
denied_retries=1,
fallback_model_settings=fallback_settings,
)
assert result is streams[1]
assert calls[0][1] is not fallback_settings
assert calls[1][1] is fallback_settings
@pytest.mark.asyncio
async def test_denial_fallback_state_round_trips_through_snapshot() -> None:
coordinator = AgentCoordinator()
await coordinator.register("root", "strix", parent_id=None)
coordinator.configure_denial_fallback("openai/gpt-5.4", 3)
await coordinator.record_denial("root")
await coordinator.mark_denial_fallback("root")
restored = AgentCoordinator()
await restored.restore(await coordinator.snapshot())
assert restored.denial_counts == {"root": 1}
assert await restored.is_on_denial_fallback("root") is True
+2
View File
@@ -28,6 +28,8 @@ def _wire_runner(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None:
timeout=300, timeout=300,
prompt_cache=True, prompt_cache=True,
extra_headers=None, extra_headers=None,
fallback_model=None,
denied_retries=3,
), ),
runtime=types.SimpleNamespace(max_context_images=3), runtime=types.SimpleNamespace(max_context_images=3),
) )
+2
View File
@@ -42,6 +42,8 @@ async def test_persistent_rate_limit_stops_gracefully(
timeout=300, timeout=300,
prompt_cache=True, prompt_cache=True,
extra_headers=None, extra_headers=None,
fallback_model=None,
denied_retries=3,
), ),
runtime=types.SimpleNamespace(max_context_images=3), runtime=types.SimpleNamespace(max_context_images=3),
) )
+2
View File
@@ -50,6 +50,8 @@ def _patch_engine_scaffold(
timeout=300, timeout=300,
prompt_cache=True, prompt_cache=True,
extra_headers=None, extra_headers=None,
fallback_model=None,
denied_retries=3,
), ),
runtime=types.SimpleNamespace(max_context_images=3), runtime=types.SimpleNamespace(max_context_images=3),
) )
+2
View File
@@ -52,6 +52,8 @@ def _settings() -> Any:
timeout=300, timeout=300,
prompt_cache=True, prompt_cache=True,
extra_headers=None, extra_headers=None,
fallback_model=None,
denied_retries=3,
), ),
runtime=types.SimpleNamespace(max_context_images=3), runtime=types.SimpleNamespace(max_context_images=3),
) )