mirror of
https://github.com/usestrix/strix.git
synced 2026-08-16 09:26:39 +02:00
feat(warmup): fail fast when a streamed request yields no tool call
OpenAI-compatible endpoints that return valid tool_calls for a non-streamed request but emit plain text (or drop the call) when streamed leave Strix's tool-driven scan unable to act. Probe the streaming path up front for custom endpoints / Ollama and abort with actionable guidance; STRIX_SKIP_TOOL_CALL_PROBE opts out.
This commit is contained in:
Alex Schapiro
@@ -40,6 +40,10 @@ class LlmSettings(BaseSettings):
|
||||
default=False,
|
||||
alias="STRIX_FORCE_REQUIRED_TOOL_CHOICE",
|
||||
)
|
||||
skip_tool_call_probe: bool = Field(
|
||||
default=False,
|
||||
alias="STRIX_SKIP_TOOL_CALL_PROBE",
|
||||
)
|
||||
prompt_cache: bool = Field(
|
||||
default=True,
|
||||
alias="STRIX_PROMPT_CACHE",
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Preflight probe: verify the model emits structured tool calls when streamed.
|
||||
|
||||
Strix is entirely tool-driven and runs every agent turn as a streamed request.
|
||||
Some OpenAI-compatible endpoints return a valid ``tool_calls`` response when a
|
||||
completion is requested non-streamed, but under streaming they emit the tool
|
||||
call as plain assistant text (or drop it entirely) and close the stream. The
|
||||
Agents SDK then sees a normal final message, the scan makes no progress, and it
|
||||
either stalls waiting for input or burns turns on empty output.
|
||||
|
||||
There is no safe client-side way to execute a tool call the endpoint never
|
||||
streamed, so we detect the missing capability up front — using the same
|
||||
streaming path the scan uses — and fail loudly with actionable guidance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from agents import ModelSettings, ModelTracing
|
||||
from agents.tool import FunctionTool
|
||||
from openai.types.responses import ResponseFunctionToolCall
|
||||
|
||||
from strix.config.models import StrixProvider
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from strix.config.settings import Settings
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_PROBE_RETRIES = 2
|
||||
|
||||
_GUIDANCE = (
|
||||
"The configured LLM endpoint did not return a structured tool call when "
|
||||
"streamed.\n\n"
|
||||
"Strix drives every action through native `tool_calls`, and it streams every "
|
||||
"turn. Some OpenAI-compatible servers return tool calls correctly for a "
|
||||
"non-streamed request but, when streamed, emit the tool call as plain text "
|
||||
"(or omit it) and end the response — so Strix can never act.\n\n"
|
||||
"Fixes:\n"
|
||||
" - llama.cpp / llama-server: start with `--jinja` so the chat template "
|
||||
"produces streamed `tool_calls` deltas.\n"
|
||||
" - Ollama: use a model whose template wires tool calling, and disable "
|
||||
"'thinking' if the template can't stream tools alongside it.\n"
|
||||
" - vLLM: set a matching `--tool-call-parser` (and `--enable-auto-tool-choice`) "
|
||||
"for the served model.\n"
|
||||
" - Other gateways: confirm streamed tool calling works for this model "
|
||||
"(a non-streamed test is not enough).\n\n"
|
||||
"If you know the endpoint streams tool calls correctly, set "
|
||||
"STRIX_SKIP_TOOL_CALL_PROBE=1 to skip this check."
|
||||
)
|
||||
|
||||
_TOOL_CONFIG_ERROR_MARKERS = (
|
||||
"jinja",
|
||||
"tool call parser",
|
||||
"tool-call-parser",
|
||||
"tool_choice",
|
||||
"does not support tools",
|
||||
"tools param",
|
||||
"tool use is not supported",
|
||||
)
|
||||
|
||||
|
||||
class ToolCallingUnsupportedError(RuntimeError):
|
||||
"""The endpoint cannot return structured tool calls over a streamed request."""
|
||||
|
||||
|
||||
async def _noop_invoke(_ctx: Any, _args: str) -> str:
|
||||
return "ok"
|
||||
|
||||
|
||||
_PROBE_TOOL = FunctionTool(
|
||||
name="strix_ready_check",
|
||||
description="Report readiness. Call this to acknowledge you can use tools.",
|
||||
params_json_schema={
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"properties": {"status": {"type": "string"}},
|
||||
"required": ["status"],
|
||||
},
|
||||
on_invoke_tool=_noop_invoke,
|
||||
strict_json_schema=True,
|
||||
)
|
||||
|
||||
_PROBE_SYSTEM = (
|
||||
"You are a setup probe. You can only respond by calling the provided tool. "
|
||||
"Do not produce any other output."
|
||||
)
|
||||
_PROBE_INPUT = 'Call the `strix_ready_check` tool now with {"status": "ok"}.'
|
||||
|
||||
|
||||
def requires_tool_call_probe(model_name: str, settings: Settings) -> bool:
|
||||
"""Only self-hosted / OpenAI-compatible routes, where the streaming leak happens."""
|
||||
return model_name.startswith("ollama/") or bool(settings.llm.api_base)
|
||||
|
||||
|
||||
def _is_tool_config_error(exc: Exception) -> bool:
|
||||
text = str(exc).lower()
|
||||
return any(marker in text for marker in _TOOL_CONFIG_ERROR_MARKERS)
|
||||
|
||||
|
||||
async def _stream_saw_tool_call(
|
||||
model_name: str, model_settings: ModelSettings, *, timeout: float | None
|
||||
) -> bool:
|
||||
model = StrixProvider().get_model(model_name)
|
||||
|
||||
async def _run() -> bool:
|
||||
stream = model.stream_response(
|
||||
system_instructions=_PROBE_SYSTEM,
|
||||
input=_PROBE_INPUT,
|
||||
model_settings=model_settings,
|
||||
tools=[_PROBE_TOOL],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
# Drain the whole stream rather than returning early: the SDK wraps it in
|
||||
# a span that must be closed in the context it was opened in.
|
||||
saw_tool_call = False
|
||||
async for event in stream:
|
||||
item = getattr(event, "item", None)
|
||||
if isinstance(item, ResponseFunctionToolCall):
|
||||
saw_tool_call = True
|
||||
response = getattr(event, "response", None)
|
||||
if response is not None and any(
|
||||
isinstance(out, ResponseFunctionToolCall)
|
||||
for out in getattr(response, "output", []) or []
|
||||
):
|
||||
saw_tool_call = True
|
||||
return saw_tool_call
|
||||
|
||||
if timeout is not None:
|
||||
return await asyncio.wait_for(_run(), timeout=timeout)
|
||||
return await _run()
|
||||
|
||||
|
||||
async def probe_tool_calling(
|
||||
model_name: str,
|
||||
settings: Settings,
|
||||
*,
|
||||
request_timeout: float | None = None,
|
||||
) -> None:
|
||||
"""Fail fast if a streamed request to ``model_name`` yields no structured tool call.
|
||||
|
||||
No-op for hosted providers and when ``STRIX_SKIP_TOOL_CALL_PROBE`` is set.
|
||||
"""
|
||||
if settings.llm.skip_tool_call_probe or not requires_tool_call_probe(model_name, settings):
|
||||
return
|
||||
|
||||
model_settings = ModelSettings(
|
||||
parallel_tool_calls=False,
|
||||
include_usage=True,
|
||||
)
|
||||
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(_PROBE_RETRIES + 1):
|
||||
try:
|
||||
saw_tool_call = await _stream_saw_tool_call(
|
||||
model_name, model_settings, timeout=request_timeout
|
||||
)
|
||||
except ToolCallingUnsupportedError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
if _is_tool_config_error(exc):
|
||||
logger.debug("Tool-call probe hit a tool-config error", exc_info=True)
|
||||
raise ToolCallingUnsupportedError(_GUIDANCE) from exc
|
||||
last_exc = exc
|
||||
logger.debug(
|
||||
"Tool-call probe attempt %d/%d failed transiently",
|
||||
attempt + 1,
|
||||
_PROBE_RETRIES + 1,
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
|
||||
if saw_tool_call:
|
||||
logger.info("Tool-call probe passed for model %s", model_name)
|
||||
return
|
||||
raise ToolCallingUnsupportedError(_GUIDANCE)
|
||||
|
||||
# All attempts raised transient errors; surface the last one unchanged so the
|
||||
# caller's existing connection-error handling reports it.
|
||||
if last_exc is not None:
|
||||
raise last_exc
|
||||
@@ -33,6 +33,7 @@ from strix.config.models import (
|
||||
)
|
||||
from strix.core.inputs import DEFAULT_MAX_TURNS
|
||||
from strix.core.paths import run_dir_for, runtime_state_dir
|
||||
from strix.core.warmup import ToolCallingUnsupportedError, probe_tool_calling
|
||||
from strix.interface.cli import run_cli
|
||||
from strix.interface.tui import run_tui
|
||||
from strix.interface.update_check import (
|
||||
@@ -422,6 +423,27 @@ async def warm_up_llm(show_model_warning: bool = True) -> None:
|
||||
)
|
||||
logger.info("LLM warm-up succeeded for dedupe model %s", dedupe_model)
|
||||
|
||||
raw_model = (llm.model or "").strip()
|
||||
await probe_tool_calling(raw_model, settings, request_timeout=llm.timeout)
|
||||
|
||||
except ToolCallingUnsupportedError as e:
|
||||
logger.debug("Tool-call probe failed", exc_info=True)
|
||||
error_text = Text()
|
||||
error_text.append("TOOL CALLING NOT SUPPORTED", style="bold red")
|
||||
error_text.append("\n\n", style="white")
|
||||
error_text.append(str(e), style="white")
|
||||
console.print("\n")
|
||||
console.print(
|
||||
Panel(
|
||||
error_text,
|
||||
title="[bold white]STRIX",
|
||||
title_align="left",
|
||||
border_style="red",
|
||||
padding=(1, 2),
|
||||
),
|
||||
)
|
||||
console.print()
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
logger.debug("LLM warm-up failed", exc_info=True)
|
||||
error_text = Text()
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import types
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
from openai.types.responses import ResponseFunctionToolCall
|
||||
|
||||
from strix.core import warmup
|
||||
from strix.core.warmup import (
|
||||
ToolCallingUnsupportedError,
|
||||
probe_tool_calling,
|
||||
requires_tool_call_probe,
|
||||
)
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
|
||||
def _settings(*, api_base: str | None = None, skip: bool = False) -> Any:
|
||||
return types.SimpleNamespace(
|
||||
llm=types.SimpleNamespace(api_base=api_base, skip_tool_call_probe=skip),
|
||||
)
|
||||
|
||||
|
||||
def _tool_call_event() -> Any:
|
||||
return types.SimpleNamespace(
|
||||
item=ResponseFunctionToolCall(
|
||||
arguments='{"status": "ok"}',
|
||||
call_id="call_1",
|
||||
name="strix_ready_check",
|
||||
type="function_call",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _text_event() -> Any:
|
||||
# A completed response whose output is a plain message, no tool call.
|
||||
return types.SimpleNamespace(
|
||||
item=None,
|
||||
response=types.SimpleNamespace(output=[types.SimpleNamespace(type="message")]),
|
||||
)
|
||||
|
||||
|
||||
class _FakeModel:
|
||||
def __init__(self, events: list[Any] | None = None, raises: Exception | None = None) -> None:
|
||||
self._events = events or []
|
||||
self._raises = raises
|
||||
|
||||
def stream_response(self, **_kwargs: Any) -> AsyncIterator[Any]:
|
||||
events = self._events
|
||||
raises = self._raises
|
||||
|
||||
async def _gen() -> AsyncIterator[Any]:
|
||||
if raises is not None:
|
||||
raise raises
|
||||
for event in events:
|
||||
yield event
|
||||
|
||||
return _gen()
|
||||
|
||||
|
||||
def _patch_model(monkeypatch: pytest.MonkeyPatch, model: _FakeModel) -> None:
|
||||
monkeypatch.setattr(
|
||||
warmup, "StrixProvider", lambda: types.SimpleNamespace(get_model=lambda _m: model)
|
||||
)
|
||||
|
||||
|
||||
def test_requires_probe_only_for_custom_endpoints_and_ollama() -> None:
|
||||
assert requires_tool_call_probe("openai/glm-5.2", _settings(api_base="http://x")) is True
|
||||
assert requires_tool_call_probe("ollama/llama3", _settings()) is True
|
||||
assert requires_tool_call_probe("openai/gpt-4o", _settings()) is False
|
||||
assert requires_tool_call_probe("anthropic/claude", _settings()) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_skipped_for_hosted_provider(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# Would raise if it tried to stream; gating must short-circuit first.
|
||||
_patch_model(monkeypatch, _FakeModel(raises=RuntimeError("should not be called")))
|
||||
await probe_tool_calling("openai/gpt-4o", _settings())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_skipped_when_setting_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_patch_model(monkeypatch, _FakeModel(raises=RuntimeError("should not be called")))
|
||||
await probe_tool_calling("openai/glm-5.2", _settings(api_base="http://x", skip=True))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_passes_on_streamed_tool_call(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_patch_model(monkeypatch, _FakeModel(events=[_tool_call_event()]))
|
||||
await probe_tool_calling("openai/glm-5.2", _settings(api_base="http://x"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_passes_when_tool_call_only_in_final_response(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
completed = types.SimpleNamespace(
|
||||
item=None,
|
||||
response=types.SimpleNamespace(
|
||||
output=[
|
||||
ResponseFunctionToolCall(
|
||||
arguments="{}",
|
||||
call_id="c",
|
||||
name="strix_ready_check",
|
||||
type="function_call",
|
||||
)
|
||||
]
|
||||
),
|
||||
)
|
||||
_patch_model(monkeypatch, _FakeModel(events=[completed]))
|
||||
await probe_tool_calling("openai/glm-5.2", _settings(api_base="http://x"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_aborts_when_only_text_streamed(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_patch_model(monkeypatch, _FakeModel(events=[_text_event()]))
|
||||
with pytest.raises(ToolCallingUnsupportedError):
|
||||
await probe_tool_calling("openai/glm-5.2", _settings(api_base="http://x"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_aborts_on_tool_config_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_patch_model(monkeypatch, _FakeModel(raises=RuntimeError("tools param requires --jinja flag")))
|
||||
with pytest.raises(ToolCallingUnsupportedError):
|
||||
await probe_tool_calling("ollama/llama3", _settings(api_base="http://x"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_retries_transient_then_passes(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
calls = {"n": 0}
|
||||
good = _FakeModel(events=[_tool_call_event()])
|
||||
|
||||
class _Flaky:
|
||||
def stream_response(self, **kwargs: Any) -> AsyncIterator[Any]:
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
|
||||
async def _boom() -> AsyncIterator[Any]:
|
||||
for _ in range(0): # make this a generator without an unreachable yield
|
||||
yield None
|
||||
raise ConnectionError("transient")
|
||||
|
||||
return _boom()
|
||||
return good.stream_response(**kwargs)
|
||||
|
||||
_patch_model(monkeypatch, _Flaky()) # type: ignore[arg-type]
|
||||
await probe_tool_calling("openai/glm-5.2", _settings(api_base="http://x"))
|
||||
assert calls["n"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_surfaces_persistent_transient_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_patch_model(monkeypatch, _FakeModel(raises=ConnectionError("down")))
|
||||
with pytest.raises(ConnectionError):
|
||||
await probe_tool_calling("openai/glm-5.2", _settings(api_base="http://x"))
|
||||
Reference in New Issue
Block a user