diff --git a/strix/config/models.py b/strix/config/models.py index 458f5c53..880e3fc6 100644 --- a/strix/config/models.py +++ b/strix/config/models.py @@ -168,11 +168,11 @@ class _NonStreamingModel(Model): Some OpenAI-compatible endpoints (notably gateways serving reasoning models) return valid ``tool_calls`` for a non-streamed completion but, when streamed, emit the tool call as plain text or drop it and close the stream — leaving - Strix's tool-driven loop with nothing to execute. For those endpoints we make - the real request non-streamed (where tool calling works) and synthesize the - minimal event sequence the runner consumes from a stream, so the rest of the - pipeline is unchanged. The only user-visible difference is no token-by-token - output. + Strix's tool-driven loop with nothing to execute. Selecting + ``STRIX_STREAM_MODE=never`` routes through this wrapper, which makes the real + request non-streamed (where tool calling works) and synthesizes the minimal + event sequence the runner consumes from a stream, so the rest of the pipeline + is unchanged. The only user-visible difference is no token-by-token output. """ def __init__(self, inner: Model) -> None: @@ -239,6 +239,10 @@ class _NonStreamingModel(Model): return self._inner.get_retry_advice(request) +def _should_run_non_streamed(settings: Settings) -> bool: + return settings.llm.stream_mode == "never" + + class StrixProvider(MultiProvider): """Route any non-OpenAI prefix through LiteLLM with the prefix preserved, so users type ``deepseek/deepseek-chat`` rather than @@ -272,9 +276,7 @@ class StrixProvider(MultiProvider): reasoning_effort=settings.llm.reasoning_effort, ) model = super().get_model(model_name) - # Custom OpenAI-compatible endpoints often stream tool calls incorrectly - # (see _NonStreamingModel); run them non-streamed unless explicitly opted in. - if settings.llm.api_base and settings.llm.stream_custom_endpoint is False: + if _should_run_non_streamed(settings): return _NonStreamingModel(model) return model diff --git a/strix/config/settings.py b/strix/config/settings.py index bd227f6c..8da857d2 100644 --- a/strix/config/settings.py +++ b/strix/config/settings.py @@ -9,6 +9,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict ReasoningEffort = Literal["none", "minimal", "low", "medium", "high", "xhigh"] +StreamMode = Literal["auto", "always", "never"] _BASE_CONFIG = SettingsConfigDict( case_sensitive=False, @@ -40,11 +41,11 @@ class LlmSettings(BaseSettings): default=False, alias="STRIX_FORCE_REQUIRED_TOOL_CHOICE", ) - # Custom OpenAI-compatible endpoints (api_base) run non-streamed by default - # because many stream tool calls incorrectly. Set to 1 to force streaming. - stream_custom_endpoint: bool = Field( - default=False, - alias="STRIX_STREAM_CUSTOM_ENDPOINT", + # auto/always stream; never runs non-streamed, for endpoints that stream tool + # calls incorrectly (some OpenAI-compatible gateways serving reasoning models). + stream_mode: StreamMode = Field( + default="auto", + alias="STRIX_STREAM_MODE", ) prompt_cache: bool = Field( default=True, diff --git a/tests/test_nonstreaming_model.py b/tests/test_nonstreaming_model.py index 73b5591c..13c08b52 100644 --- a/tests/test_nonstreaming_model.py +++ b/tests/test_nonstreaming_model.py @@ -52,12 +52,12 @@ class _FakeModel(Model): raise AssertionError("inner stream_response must never be called") -def _settings(*, api_base: str | None, stream_custom_endpoint: bool = False) -> SimpleNamespace: +def _settings(*, api_base: str | None, stream_mode: str = "auto") -> SimpleNamespace: return SimpleNamespace( llm=SimpleNamespace( model="openai/glm", api_base=api_base, - stream_custom_endpoint=stream_custom_endpoint, + stream_mode=stream_mode, reasoning_effort="high", ) ) @@ -131,7 +131,7 @@ async def test_stream_response_delegates_get_response() -> None: assert inner.get_response_calls == 1 -def test_get_model_wraps_custom_endpoint() -> None: +def test_get_model_auto_streams_custom_endpoint() -> None: sentinel = _FakeModel(ModelResponse(output=[], usage=Usage(), response_id=None)) with ( patch("strix.config.models.load_settings", return_value=_settings(api_base="http://x/v1")), @@ -141,10 +141,10 @@ def test_get_model_wraps_custom_endpoint() -> None: ), ): model = StrixProvider().get_model("openai/glm") - assert isinstance(model, _NonStreamingModel) + assert model is sentinel -def test_get_model_hosted_stays_streamed() -> None: +def test_get_model_auto_streams_hosted() -> None: sentinel = _FakeModel(ModelResponse(output=[], usage=Usage(), response_id=None)) with ( patch("strix.config.models.load_settings", return_value=_settings(api_base="")), @@ -157,12 +157,12 @@ def test_get_model_hosted_stays_streamed() -> None: assert model is sentinel -def test_get_model_opt_out_keeps_streaming() -> None: +def test_get_model_stream_mode_never_wraps() -> None: sentinel = _FakeModel(ModelResponse(output=[], usage=Usage(), response_id=None)) with ( patch( "strix.config.models.load_settings", - return_value=_settings(api_base="http://x/v1", stream_custom_endpoint=True), + return_value=_settings(api_base="http://x/v1", stream_mode="never"), ), patch( "agents.models.multi_provider.MultiProvider.get_model", @@ -170,4 +170,4 @@ def test_get_model_opt_out_keeps_streaming() -> None: ), ): model = StrixProvider().get_model("openai/glm") - assert model is sentinel + assert isinstance(model, _NonStreamingModel)