Sign in with a ChatGPT subscription for inference (#854)

Co-authored-by: Jonathan Singer <jonathansinger@Jonathans-MacBook-Pro.local>
Co-authored-by: Jonathan Singer <jonathansinger@Mac-3004.lan>
Co-authored-by: Ahmed Allam <ahmed39652003@gmail.com>

tests/test_codex_streaming.py

@@ -0,0 +1,225 @@
+"""Regression test for the ChatGPT Codex backend's streaming requirement.
+
+The backend rejects non-streamed requests with ``{"detail": "Stream must be set
+to true"}``. ``_CodexResponsesModel`` must therefore issue a streamed request
+even from the non-streaming ``get_response`` path and aggregate the events into
+a single response. A local server that mimics that behaviour proves the wrapper
+works where the stock responses model would fail.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from typing import TYPE_CHECKING, Any
+
+import pytest
+from agents.model_settings import ModelSettings
+from agents.models.interface import ModelTracing
+from agents.models.openai_responses import OpenAIResponsesModel
+from openai import AsyncOpenAI, BadRequestError
+from openai.types.responses import ResponseOutputMessage, ResponseOutputText
+
+from strix.config import codex
+from strix.config.models import _CodexResponsesModel
+
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator, Iterator
+
+
+def _response_payload() -> dict[str, Any]:
+    return {
+        "id": "resp_1",
+        "object": "response",
+        "created_at": 0,
+        "status": "completed",
+        "model": "gpt-5.5",
+        "output": [
+            {
+                "type": "message",
+                "id": "m1",
+                "status": "completed",
+                "role": "assistant",
+                "content": [{"type": "output_text", "text": "OK", "annotations": []}],
+            }
+        ],
+        "usage": {
+            "input_tokens": 1,
+            "output_tokens": 1,
+            "total_tokens": 2,
+            "input_tokens_details": {"cached_tokens": 0},
+            "output_tokens_details": {"reasoning_tokens": 0},
+        },
+        "parallel_tool_calls": False,
+        "tool_choice": "auto",
+        "tools": [],
+        "metadata": {},
+        "temperature": 1.0,
+        "top_p": 1.0,
+        "error": None,
+        "incomplete_details": None,
+        "instructions": None,
+        "max_output_tokens": None,
+    }
+
+
+_CAPTURED: dict[str, Any] = {}
+
+
+class _Handler(BaseHTTPRequestHandler):
+    def log_message(self, *args: Any) -> None:
+        pass
+
+    def do_POST(self) -> None:
+        length = int(self.headers.get("Content-Length", 0))
+        body = json.loads(self.rfile.read(length) or b"{}")
+        _CAPTURED.clear()
+        _CAPTURED.update(body)
+        if not body.get("stream"):
+            payload = json.dumps({"detail": "Stream must be set to true"}).encode()
+            self.send_response(400)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(payload)))
+            self.end_headers()
+            self.wfile.write(payload)
+            return
+        event = {
+            "type": "response.completed",
+            "sequence_number": 0,
+            "response": _response_payload(),
+        }
+        frame = f"event: response.completed\ndata: {json.dumps(event)}\n\n".encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "text/event-stream")
+        self.end_headers()
+        self.wfile.write(frame)
+
+
+@pytest.fixture
+def backend_url() -> Iterator[str]:
+    server = HTTPServer(("127.0.0.1", 0), _Handler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    try:
+        yield f"http://127.0.0.1:{server.server_address[1]}/backend-api/codex"
+    finally:
+        server.shutdown()
+        server.server_close()
+
+
+def _client(base_url: str) -> AsyncOpenAI:
+    return AsyncOpenAI(api_key="tok", base_url=base_url)
+
+
+def _call_kwargs() -> dict[str, Any]:
+    return {
+        "system_instructions": "s",
+        "input": "hi",
+        "model_settings": ModelSettings(
+            store=False, response_include=["reasoning.encrypted_content"]
+        ),
+        "tools": [],
+        "output_schema": None,
+        "handoffs": [],
+        "tracing": ModelTracing.DISABLED,
+        "previous_response_id": None,
+        "conversation_id": None,
+        "prompt": None,
+    }
+
+
+@pytest.mark.asyncio
+async def test_stock_model_fails_on_non_streamed_backend(backend_url: str) -> None:
+    model = OpenAIResponsesModel(model="gpt-5.5", openai_client=_client(backend_url))
+    with pytest.raises(BadRequestError, match="Stream must be set to true"):
+        await model.get_response(**_call_kwargs())
+
+
+@pytest.mark.asyncio
+async def test_codex_model_streams_and_aggregates(backend_url: str) -> None:
+    model = _CodexResponsesModel(model="gpt-5.5", openai_client=_client(backend_url))
+    response = await model.get_response(**_call_kwargs())
+    message = response.output[0]
+    assert isinstance(message, ResponseOutputMessage)
+    text = message.content[0]
+    assert isinstance(text, ResponseOutputText)
+    assert text.text == "OK"
+    assert response.usage.total_tokens == 2
+
+
+class _TrackingStream:
+    """An async iterator that yields, then raises, and records if it was closed."""
+
+    def __init__(self, events: list[Any], error: Exception | None) -> None:
+        self._events = iter(events)
+        self._error = error
+        self.closed = False
+
+    def __aiter__(self) -> _TrackingStream:
+        return self
+
+    async def __anext__(self) -> Any:
+        try:
+            return next(self._events)
+        except StopIteration:
+            if self._error is not None:
+                raise self._error from None
+            raise StopAsyncIteration from None
+
+    async def aclose(self) -> None:
+        self.closed = True
+
+
+async def _drain(gen: AsyncIterator[Any]) -> list[Any]:
+    return [event async for event in gen]
+
+
+@pytest.mark.asyncio
+async def test_guarded_converts_guardrail_error() -> None:
+    # A mid-stream backend rejection becomes a typed, model-tagged error.
+    model = _CodexResponsesModel(model="gpt-5.6-sol", openai_client=_client("http://x/backend-api"))
+    guardrail = RuntimeError("This content was flagged for possible cybersecurity risk.")
+    stream = _TrackingStream(["a", "b"], guardrail)
+    with pytest.raises(codex.CodexContentGuardrailError) as exc_info:
+        await _drain(model._guarded(stream))
+    assert exc_info.value.model == "gpt-5.6-sol"
+    assert stream.closed is True  # underlying stream is released
+
+
+@pytest.mark.asyncio
+async def test_guarded_passes_through_other_errors() -> None:
+    # A non-guardrail error propagates unchanged (still not swallowed).
+    model = _CodexResponsesModel(model="gpt-5.5", openai_client=_client("http://x/backend-api"))
+    boom = RuntimeError("some unrelated failure")
+    stream = _TrackingStream(["a"], boom)
+    with pytest.raises(RuntimeError, match="some unrelated failure"):
+        await _drain(model._guarded(stream))
+    assert stream.closed is True
+
+
+@pytest.mark.asyncio
+async def test_guarded_yields_all_events_when_clean() -> None:
+    model = _CodexResponsesModel(model="gpt-5.4", openai_client=_client("http://x/backend-api"))
+    stream = _TrackingStream(["a", "b", "c"], None)
+    assert await _drain(model._guarded(stream)) == ["a", "b", "c"]
+    assert stream.closed is True
+
+
+@pytest.mark.asyncio
+async def test_codex_model_self_enforces_backend_requirements(backend_url: str) -> None:
+    # The caller passes ordinary settings; the model must impose the backend's
+    # requirements (stream, store=false, encrypted reasoning) and the configured
+    # reasoning effort itself.
+    model = _CodexResponsesModel(
+        model="gpt-5.4", openai_client=_client(backend_url), reasoning_effort="high"
+    )
+    kwargs = _call_kwargs()
+    kwargs["model_settings"] = ModelSettings()  # nothing special from the caller
+    await model.get_response(**kwargs)
+
+    assert _CAPTURED["stream"] is True
+    assert _CAPTURED["store"] is False
+    assert _CAPTURED["include"] == ["reasoning.encrypted_content"]
+    assert _CAPTURED["reasoning"] == {"effort": "high"}