From 0e28732d2983656de013caebfc750ed71ce693ef Mon Sep 17 00:00:00 2001 From: Alex Schapiro Date: Wed, 29 Jul 2026 14:17:17 +0000 Subject: [PATCH] feat(warmup): fail fast when a streamed request yields no tool call OpenAI-compatible endpoints that return valid tool_calls for a non-streamed request but emit plain text (or drop the call) when streamed leave Strix's tool-driven scan unable to act. Probe the streaming path up front for custom endpoints / Ollama and abort with actionable guidance; STRIX_SKIP_TOOL_CALL_PROBE opts out. --- strix/config/settings.py | 4 + strix/core/warmup.py | 190 +++++++++++++++++++++++++++++++++++++ strix/interface/main.py | 22 +++++ tests/test_warmup_probe.py | 158 ++++++++++++++++++++++++++++++ 4 files changed, 374 insertions(+) create mode 100644 strix/core/warmup.py create mode 100644 tests/test_warmup_probe.py diff --git a/strix/config/settings.py b/strix/config/settings.py index 78d52273..cf04b05a 100644 --- a/strix/config/settings.py +++ b/strix/config/settings.py @@ -40,6 +40,10 @@ class LlmSettings(BaseSettings): default=False, alias="STRIX_FORCE_REQUIRED_TOOL_CHOICE", ) + skip_tool_call_probe: bool = Field( + default=False, + alias="STRIX_SKIP_TOOL_CALL_PROBE", + ) prompt_cache: bool = Field( default=True, alias="STRIX_PROMPT_CACHE", diff --git a/strix/core/warmup.py b/strix/core/warmup.py new file mode 100644 index 00000000..633cdf0a --- /dev/null +++ b/strix/core/warmup.py @@ -0,0 +1,190 @@ +"""Preflight probe: verify the model emits structured tool calls when streamed. + +Strix is entirely tool-driven and runs every agent turn as a streamed request. +Some OpenAI-compatible endpoints return a valid ``tool_calls`` response when a +completion is requested non-streamed, but under streaming they emit the tool +call as plain assistant text (or drop it entirely) and close the stream. The +Agents SDK then sees a normal final message, the scan makes no progress, and it +either stalls waiting for input or burns turns on empty output. + +There is no safe client-side way to execute a tool call the endpoint never +streamed, so we detect the missing capability up front — using the same +streaming path the scan uses — and fail loudly with actionable guidance. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import TYPE_CHECKING, Any + +from agents import ModelSettings, ModelTracing +from agents.tool import FunctionTool +from openai.types.responses import ResponseFunctionToolCall + +from strix.config.models import StrixProvider + + +if TYPE_CHECKING: + from strix.config.settings import Settings + + +logger = logging.getLogger(__name__) + +_PROBE_RETRIES = 2 + +_GUIDANCE = ( + "The configured LLM endpoint did not return a structured tool call when " + "streamed.\n\n" + "Strix drives every action through native `tool_calls`, and it streams every " + "turn. Some OpenAI-compatible servers return tool calls correctly for a " + "non-streamed request but, when streamed, emit the tool call as plain text " + "(or omit it) and end the response — so Strix can never act.\n\n" + "Fixes:\n" + " - llama.cpp / llama-server: start with `--jinja` so the chat template " + "produces streamed `tool_calls` deltas.\n" + " - Ollama: use a model whose template wires tool calling, and disable " + "'thinking' if the template can't stream tools alongside it.\n" + " - vLLM: set a matching `--tool-call-parser` (and `--enable-auto-tool-choice`) " + "for the served model.\n" + " - Other gateways: confirm streamed tool calling works for this model " + "(a non-streamed test is not enough).\n\n" + "If you know the endpoint streams tool calls correctly, set " + "STRIX_SKIP_TOOL_CALL_PROBE=1 to skip this check." +) + +_TOOL_CONFIG_ERROR_MARKERS = ( + "jinja", + "tool call parser", + "tool-call-parser", + "tool_choice", + "does not support tools", + "tools param", + "tool use is not supported", +) + + +class ToolCallingUnsupportedError(RuntimeError): + """The endpoint cannot return structured tool calls over a streamed request.""" + + +async def _noop_invoke(_ctx: Any, _args: str) -> str: + return "ok" + + +_PROBE_TOOL = FunctionTool( + name="strix_ready_check", + description="Report readiness. Call this to acknowledge you can use tools.", + params_json_schema={ + "type": "object", + "additionalProperties": False, + "properties": {"status": {"type": "string"}}, + "required": ["status"], + }, + on_invoke_tool=_noop_invoke, + strict_json_schema=True, +) + +_PROBE_SYSTEM = ( + "You are a setup probe. You can only respond by calling the provided tool. " + "Do not produce any other output." +) +_PROBE_INPUT = 'Call the `strix_ready_check` tool now with {"status": "ok"}.' + + +def requires_tool_call_probe(model_name: str, settings: Settings) -> bool: + """Only self-hosted / OpenAI-compatible routes, where the streaming leak happens.""" + return model_name.startswith("ollama/") or bool(settings.llm.api_base) + + +def _is_tool_config_error(exc: Exception) -> bool: + text = str(exc).lower() + return any(marker in text for marker in _TOOL_CONFIG_ERROR_MARKERS) + + +async def _stream_saw_tool_call( + model_name: str, model_settings: ModelSettings, *, timeout: float | None +) -> bool: + model = StrixProvider().get_model(model_name) + + async def _run() -> bool: + stream = model.stream_response( + system_instructions=_PROBE_SYSTEM, + input=_PROBE_INPUT, + model_settings=model_settings, + tools=[_PROBE_TOOL], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + previous_response_id=None, + conversation_id=None, + prompt=None, + ) + # Drain the whole stream rather than returning early: the SDK wraps it in + # a span that must be closed in the context it was opened in. + saw_tool_call = False + async for event in stream: + item = getattr(event, "item", None) + if isinstance(item, ResponseFunctionToolCall): + saw_tool_call = True + response = getattr(event, "response", None) + if response is not None and any( + isinstance(out, ResponseFunctionToolCall) + for out in getattr(response, "output", []) or [] + ): + saw_tool_call = True + return saw_tool_call + + if timeout is not None: + return await asyncio.wait_for(_run(), timeout=timeout) + return await _run() + + +async def probe_tool_calling( + model_name: str, + settings: Settings, + *, + request_timeout: float | None = None, +) -> None: + """Fail fast if a streamed request to ``model_name`` yields no structured tool call. + + No-op for hosted providers and when ``STRIX_SKIP_TOOL_CALL_PROBE`` is set. + """ + if settings.llm.skip_tool_call_probe or not requires_tool_call_probe(model_name, settings): + return + + model_settings = ModelSettings( + parallel_tool_calls=False, + include_usage=True, + ) + + last_exc: Exception | None = None + for attempt in range(_PROBE_RETRIES + 1): + try: + saw_tool_call = await _stream_saw_tool_call( + model_name, model_settings, timeout=request_timeout + ) + except ToolCallingUnsupportedError: + raise + except Exception as exc: + if _is_tool_config_error(exc): + logger.debug("Tool-call probe hit a tool-config error", exc_info=True) + raise ToolCallingUnsupportedError(_GUIDANCE) from exc + last_exc = exc + logger.debug( + "Tool-call probe attempt %d/%d failed transiently", + attempt + 1, + _PROBE_RETRIES + 1, + exc_info=True, + ) + continue + + if saw_tool_call: + logger.info("Tool-call probe passed for model %s", model_name) + return + raise ToolCallingUnsupportedError(_GUIDANCE) + + # All attempts raised transient errors; surface the last one unchanged so the + # caller's existing connection-error handling reports it. + if last_exc is not None: + raise last_exc diff --git a/strix/interface/main.py b/strix/interface/main.py index 0fbcbf48..31d5b36e 100644 --- a/strix/interface/main.py +++ b/strix/interface/main.py @@ -33,6 +33,7 @@ from strix.config.models import ( ) from strix.core.inputs import DEFAULT_MAX_TURNS from strix.core.paths import run_dir_for, runtime_state_dir +from strix.core.warmup import ToolCallingUnsupportedError, probe_tool_calling from strix.interface.cli import run_cli from strix.interface.tui import run_tui from strix.interface.update_check import ( @@ -422,6 +423,27 @@ async def warm_up_llm(show_model_warning: bool = True) -> None: ) logger.info("LLM warm-up succeeded for dedupe model %s", dedupe_model) + raw_model = (llm.model or "").strip() + await probe_tool_calling(raw_model, settings, request_timeout=llm.timeout) + + except ToolCallingUnsupportedError as e: + logger.debug("Tool-call probe failed", exc_info=True) + error_text = Text() + error_text.append("TOOL CALLING NOT SUPPORTED", style="bold red") + error_text.append("\n\n", style="white") + error_text.append(str(e), style="white") + console.print("\n") + console.print( + Panel( + error_text, + title="[bold white]STRIX", + title_align="left", + border_style="red", + padding=(1, 2), + ), + ) + console.print() + sys.exit(1) except Exception as e: logger.debug("LLM warm-up failed", exc_info=True) error_text = Text() diff --git a/tests/test_warmup_probe.py b/tests/test_warmup_probe.py new file mode 100644 index 00000000..0ebc74e6 --- /dev/null +++ b/tests/test_warmup_probe.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import types +from typing import TYPE_CHECKING, Any + +import pytest +from openai.types.responses import ResponseFunctionToolCall + +from strix.core import warmup +from strix.core.warmup import ( + ToolCallingUnsupportedError, + probe_tool_calling, + requires_tool_call_probe, +) + + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + +def _settings(*, api_base: str | None = None, skip: bool = False) -> Any: + return types.SimpleNamespace( + llm=types.SimpleNamespace(api_base=api_base, skip_tool_call_probe=skip), + ) + + +def _tool_call_event() -> Any: + return types.SimpleNamespace( + item=ResponseFunctionToolCall( + arguments='{"status": "ok"}', + call_id="call_1", + name="strix_ready_check", + type="function_call", + ), + ) + + +def _text_event() -> Any: + # A completed response whose output is a plain message, no tool call. + return types.SimpleNamespace( + item=None, + response=types.SimpleNamespace(output=[types.SimpleNamespace(type="message")]), + ) + + +class _FakeModel: + def __init__(self, events: list[Any] | None = None, raises: Exception | None = None) -> None: + self._events = events or [] + self._raises = raises + + def stream_response(self, **_kwargs: Any) -> AsyncIterator[Any]: + events = self._events + raises = self._raises + + async def _gen() -> AsyncIterator[Any]: + if raises is not None: + raise raises + for event in events: + yield event + + return _gen() + + +def _patch_model(monkeypatch: pytest.MonkeyPatch, model: _FakeModel) -> None: + monkeypatch.setattr( + warmup, "StrixProvider", lambda: types.SimpleNamespace(get_model=lambda _m: model) + ) + + +def test_requires_probe_only_for_custom_endpoints_and_ollama() -> None: + assert requires_tool_call_probe("openai/glm-5.2", _settings(api_base="http://x")) is True + assert requires_tool_call_probe("ollama/llama3", _settings()) is True + assert requires_tool_call_probe("openai/gpt-4o", _settings()) is False + assert requires_tool_call_probe("anthropic/claude", _settings()) is False + + +@pytest.mark.asyncio +async def test_probe_skipped_for_hosted_provider(monkeypatch: pytest.MonkeyPatch) -> None: + # Would raise if it tried to stream; gating must short-circuit first. + _patch_model(monkeypatch, _FakeModel(raises=RuntimeError("should not be called"))) + await probe_tool_calling("openai/gpt-4o", _settings()) + + +@pytest.mark.asyncio +async def test_probe_skipped_when_setting_disabled(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_model(monkeypatch, _FakeModel(raises=RuntimeError("should not be called"))) + await probe_tool_calling("openai/glm-5.2", _settings(api_base="http://x", skip=True)) + + +@pytest.mark.asyncio +async def test_probe_passes_on_streamed_tool_call(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_model(monkeypatch, _FakeModel(events=[_tool_call_event()])) + await probe_tool_calling("openai/glm-5.2", _settings(api_base="http://x")) + + +@pytest.mark.asyncio +async def test_probe_passes_when_tool_call_only_in_final_response( + monkeypatch: pytest.MonkeyPatch, +) -> None: + completed = types.SimpleNamespace( + item=None, + response=types.SimpleNamespace( + output=[ + ResponseFunctionToolCall( + arguments="{}", + call_id="c", + name="strix_ready_check", + type="function_call", + ) + ] + ), + ) + _patch_model(monkeypatch, _FakeModel(events=[completed])) + await probe_tool_calling("openai/glm-5.2", _settings(api_base="http://x")) + + +@pytest.mark.asyncio +async def test_probe_aborts_when_only_text_streamed(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_model(monkeypatch, _FakeModel(events=[_text_event()])) + with pytest.raises(ToolCallingUnsupportedError): + await probe_tool_calling("openai/glm-5.2", _settings(api_base="http://x")) + + +@pytest.mark.asyncio +async def test_probe_aborts_on_tool_config_error(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_model(monkeypatch, _FakeModel(raises=RuntimeError("tools param requires --jinja flag"))) + with pytest.raises(ToolCallingUnsupportedError): + await probe_tool_calling("ollama/llama3", _settings(api_base="http://x")) + + +@pytest.mark.asyncio +async def test_probe_retries_transient_then_passes(monkeypatch: pytest.MonkeyPatch) -> None: + calls = {"n": 0} + good = _FakeModel(events=[_tool_call_event()]) + + class _Flaky: + def stream_response(self, **kwargs: Any) -> AsyncIterator[Any]: + calls["n"] += 1 + if calls["n"] == 1: + + async def _boom() -> AsyncIterator[Any]: + for _ in range(0): # make this a generator without an unreachable yield + yield None + raise ConnectionError("transient") + + return _boom() + return good.stream_response(**kwargs) + + _patch_model(monkeypatch, _Flaky()) # type: ignore[arg-type] + await probe_tool_calling("openai/glm-5.2", _settings(api_base="http://x")) + assert calls["n"] == 2 + + +@pytest.mark.asyncio +async def test_probe_surfaces_persistent_transient_error(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_model(monkeypatch, _FakeModel(raises=ConnectionError("down"))) + with pytest.raises(ConnectionError): + await probe_tool_calling("openai/glm-5.2", _settings(api_base="http://x"))