mirror of
https://github.com/usestrix/strix.git
synced 2026-08-16 09:26:39 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fac2fd1100 | ||
|
|
3c38cb453d | ||
|
|
b69af37cb2 | ||
|
|
72cb15a20a | ||
|
|
97336d53e4 | ||
|
|
0abe82d622 | ||
|
|
6735a6f89e | ||
|
|
8bd6c8e87a | ||
|
|
68ea6fca65 | ||
|
|
657aa5cbe6 | ||
|
|
82dcd31357 | ||
|
|
6719a70611 | ||
|
|
ea6d53f4e9 | ||
|
|
3bcf3778f0 | ||
|
|
4a455b1e62 | ||
|
|
6f70b6f319 |
@@ -21,7 +21,7 @@ repos:
|
||||
fastapi,
|
||||
pytest,
|
||||
hatchling,
|
||||
"openai-agents[litellm]==0.14.6",
|
||||
"openai-agents[litellm]>=0.19.0,<0.20",
|
||||
]
|
||||
args: [--install-types, --non-interactive]
|
||||
|
||||
|
||||
@@ -185,6 +185,28 @@ strix --target https://github.com/org/repo
|
||||
strix --target https://your-app.com
|
||||
```
|
||||
|
||||
### API Testing (OpenAPI / Swagger / Postman)
|
||||
|
||||
Point Strix at an API contract and it tests every declared endpoint instead of
|
||||
having to discover them by crawling. Pair the spec with the live base URL so the
|
||||
agent knows where to send traffic:
|
||||
|
||||
```bash
|
||||
# OpenAPI / Swagger file (.json / .yaml)
|
||||
strix --target ./openapi.yaml --target https://api.your-app.com
|
||||
|
||||
# Postman collection export
|
||||
strix --target ./collection.postman_collection.json --target https://api.your-app.com
|
||||
|
||||
# Postman collection pulled live by id (no manual export)
|
||||
export POSTMAN_API_KEY="PMAK-..."
|
||||
strix --target postman://<collection-uuid>
|
||||
|
||||
# ...with a Postman environment to resolve {{baseUrl}} / token variables
|
||||
strix --target "postman://<collection-uuid>?env=<environment-uuid>"
|
||||
```
|
||||
|
||||
|
||||
### Advanced Testing Scenarios
|
||||
|
||||
```bash
|
||||
|
||||
@@ -16,7 +16,8 @@ RUN mkdir -p /out/bin && \
|
||||
go install -v github.com/projectdiscovery/katana/cmd/katana@latest && \
|
||||
go install -v github.com/projectdiscovery/cvemap/cmd/vulnx@latest && \
|
||||
go install -v github.com/jaeles-project/gospider@latest && \
|
||||
go install -v github.com/projectdiscovery/interactsh/cmd/interactsh-client@latest
|
||||
go install -v github.com/projectdiscovery/interactsh/cmd/interactsh-client@latest && \
|
||||
go install -v golang.org/x/vuln/cmd/govulncheck@latest
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runtime stage
|
||||
@@ -53,6 +54,7 @@ RUN apt-get update && \
|
||||
nmap ncat ndiff \
|
||||
sqlmap nuclei subfinder naabu ffuf \
|
||||
nodejs npm pipx \
|
||||
golang-go \
|
||||
libcap2-bin \
|
||||
gdb \
|
||||
libnss3-tools \
|
||||
|
||||
@@ -80,6 +80,10 @@ affecting the agents that do the actual testing.
|
||||
API key for Perplexity AI. Enables real-time web search during scans for OSINT and vulnerability research.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="POSTMAN_API_KEY" type="string">
|
||||
Postman API key (`PMAK-…`). Enables fetching Postman collections by id as a target (`postman://<collection-uid>`), and Postman environments (`postman://<collection-uid>?env=<environment-uid>`) to resolve collection variables. Not needed when passing a local collection export file.
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="STRIX_TELEMETRY" default="1" type="string">
|
||||
Telemetry toggle. Set to `0`, `false`, `no`, or `off` to disable telemetry (PostHog, Scarf, OTEL).
|
||||
</ParamField>
|
||||
|
||||
+13
-1
@@ -12,11 +12,17 @@ strix (--target <target> | --target-list <path>) [options]
|
||||
## Options
|
||||
|
||||
<ParamField path="--target, -t" type="string">
|
||||
Target to test. Accepts URLs, repositories, local directories, domains, or IP addresses. Can be specified multiple times. Fresh runs require at least one target source: `--target` or `--target-list`.
|
||||
Target to test. Accepts URLs, repositories, local directories, domains, IP addresses, API spec files (OpenAPI/Swagger `.json`/`.yaml`, a Postman collection export), or a live Postman collection by id (`postman://<collection-uuid>`). Can be specified multiple times. Fresh runs require at least one target source: `--target` or `--target-list`.
|
||||
|
||||
When the target is an API spec, Strix copies it into the agent's workspace and authorizes the base URLs it declares (including those resolved from a Postman environment) as in-scope hosts - so the agent reads the contract and tests the full declared surface instead of discovering endpoints by crawling. Pair the spec with the deployed base URL (e.g. `--target ./openapi.yaml --target https://api.example.com`) so the agent has a reachable host to attack.
|
||||
|
||||
<Note>
|
||||
A local directory is mounted into the sandbox live and **writable**, so the agent edits your real files (`.git` excepted). Commit or stash first.
|
||||
</Note>
|
||||
|
||||
<Note>
|
||||
Fetching a Postman collection by id requires `POSTMAN_API_KEY`. Add `?env=<environment-uuid>` to also pull a Postman environment, which resolves `{{baseUrl}}` / token variables the collection references (e.g. `postman://<collection-uuid>?env=<environment-uid>`).
|
||||
</Note>
|
||||
</ParamField>
|
||||
|
||||
<ParamField path="--target-list" type="string">
|
||||
@@ -128,6 +134,12 @@ strix -n --target ./ --scan-mode quick --scope-mode diff --diff-base origin/main
|
||||
# Multi-target white-box testing
|
||||
strix -t https://github.com/org/app -t https://staging.example.com
|
||||
|
||||
# API spec + live target (OpenAPI/Swagger file or Postman collection)
|
||||
strix -t ./openapi.yaml -t https://api.example.com
|
||||
|
||||
# Postman collection pulled live by id (+ optional environment)
|
||||
strix -t "postman://<collection-uuid>?env=<environment-uuid>"
|
||||
|
||||
# Targets from a file
|
||||
strix --target-list ./targets.txt
|
||||
```
|
||||
|
||||
+9
-2
@@ -33,8 +33,8 @@ classifiers = [
|
||||
"Programming Language :: Python :: 3.14",
|
||||
]
|
||||
dependencies = [
|
||||
"openai-agents[litellm]==0.14.6",
|
||||
"openai>=2.26.0,<2.45",
|
||||
"openai-agents[litellm]>=0.19.0,<0.20",
|
||||
"openai>=2.45.0,<3",
|
||||
"litellm",
|
||||
"pydantic>=2.11.3",
|
||||
"pydantic-settings>=2.13.0",
|
||||
@@ -48,6 +48,7 @@ dependencies = [
|
||||
# Cap <49: 49.x drops the universal2 macOS wheel (arm64-only), which breaks
|
||||
# the Intel macOS (macos-x86_64) release build's `uv sync --frozen`.
|
||||
"cryptography>=48.0.1,<49",
|
||||
"pyyaml>=6.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
@@ -67,6 +68,7 @@ dev = [
|
||||
"pyinstaller>=6.17.0; python_version >= '3.12' and python_version < '3.15'",
|
||||
"pytest>=8.3",
|
||||
"pytest-asyncio>=0.24",
|
||||
"types-requests>=2.32",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
@@ -132,6 +134,7 @@ module = [
|
||||
"pydantic_settings.*",
|
||||
"reportlab.*",
|
||||
"pypdf.*",
|
||||
"yaml.*",
|
||||
"pygments.*",
|
||||
]
|
||||
ignore_missing_imports = true
|
||||
@@ -233,6 +236,10 @@ ignore = [
|
||||
"strix/interface/auth_cli.py" = ["N802"]
|
||||
"tests/test_codex_streaming.py" = ["N802"]
|
||||
"tests/test_disable_streaming.py" = ["N802"]
|
||||
"tests/test_tool_call_ids.py" = ["N802"]
|
||||
"tests/test_tool_call_limits.py" = ["N802", "SLF001"]
|
||||
"tests/test_stream_idle_timeout.py" = ["N802", "SLF001"]
|
||||
"tests/test_unknown_tool_recovery.py" = ["N802"]
|
||||
"tests/test_report_pdf.py" = ["S105", "S106"]
|
||||
# Stdlib HTTP handler overrides (do_GET/do_POST) and lazy imports that avoid a
|
||||
# circular dependency with strix.telemetry / strix.interface.viewer.report_pdf.
|
||||
|
||||
@@ -559,7 +559,7 @@ def registered_agent_tools() -> tuple[Tool, ...]:
|
||||
|
||||
def build_strix_agent(
|
||||
*,
|
||||
name: str = "strix",
|
||||
name: str = "agent",
|
||||
skills: list[str] | None = None,
|
||||
is_root: bool,
|
||||
scan_mode: str = "deep",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
You are Strix, an advanced AI application security validation agent developed by OmniSecure Labs. Your purpose is to perform authorized security verification, reproduce and validate weaknesses on in-scope assets, and help remediate real security issues.
|
||||
You are an advanced AI application security validation agent. Your purpose is to perform authorized security verification, reproduce and validate weaknesses on in-scope assets, and help remediate real security issues.
|
||||
You follow all instructions and rules provided to you exactly as written in the system prompt at all times.
|
||||
{% if is_root %}
|
||||
<root_agent_directive>
|
||||
@@ -22,12 +22,13 @@ CLI OUTPUT:
|
||||
- You may use simple markdown: **bold**, *italic*, `code`, ~~strikethrough~~, [links](url), and # headers
|
||||
- Do NOT use complex markdown like bullet lists, numbered lists, or tables
|
||||
- Use line breaks and indentation for structure
|
||||
- NEVER use "Strix" or any identifiable names/markers in HTTP requests, payloads, user-agents, or any inputs
|
||||
- NEVER use any identifiable names/markers in HTTP requests, payloads, user-agents, or any inputs
|
||||
|
||||
INTER-AGENT MESSAGES:
|
||||
- Messages from other agents arrive prefixed with a header like `[Message from agent <name> | type=... | priority=...]`. Treat them as internal context — never repeat them verbatim in your own output.
|
||||
- Treat agent identity / inherited-context preambles as internal metadata; do not echo them in outputs or tool calls.
|
||||
- Minimize inter-agent messaging: only message when essential for coordination or assistance; avoid routine status updates; batch non-urgent information; prefer parent/child completion flows and shared artifacts over messaging
|
||||
- wait_for_agents blocks and resumes you automatically, so it is never a poll you repeat: issue exactly ONE wait, then stop and react to what it returns. Never write out a wait/check loop (wait → view_agent_graph → wait → ...) ahead of time — those extra calls only strand you and are collapsed anyway
|
||||
|
||||
{% if interactive %}
|
||||
INTERACTIVE BEHAVIOR:
|
||||
@@ -57,7 +58,7 @@ AUTONOMOUS BEHAVIOR:
|
||||
<execution_guidelines>
|
||||
{% if system_prompt_context and system_prompt_context.authorized_targets %}
|
||||
SYSTEM-VERIFIED SCOPE:
|
||||
- The following scope metadata is injected by the Strix platform into the system prompt and is authoritative
|
||||
- The following scope metadata is injected by the platform into the system prompt and is authoritative
|
||||
- Scope source: {{ system_prompt_context.scope_source }}
|
||||
- Authorization source: {{ system_prompt_context.authorization_source }}
|
||||
- Every target listed below has already been verified by the platform as in-scope and authorized
|
||||
|
||||
+194
-7
@@ -2,11 +2,14 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import inspect
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from agents import (
|
||||
set_default_openai_api,
|
||||
@@ -24,12 +27,19 @@ from agents.retry import (
|
||||
RetryPolicyContext,
|
||||
retry_policies,
|
||||
)
|
||||
from openai.types.responses import Response, ResponseCompletedEvent
|
||||
from openai.types.responses import (
|
||||
Response,
|
||||
ResponseCompletedEvent,
|
||||
ResponseOutputItemAddedEvent,
|
||||
ResponseOutputItemDoneEvent,
|
||||
)
|
||||
from openai.types.responses.response_usage import ResponseUsage
|
||||
from openai.types.shared import Reasoning
|
||||
|
||||
from strix.config import codex
|
||||
from strix.config.loader import load_settings
|
||||
from strix.config.tool_call_ids import TurnCallIdRewriter, dedupe_input
|
||||
from strix.config.tool_call_limits import TurnToolCallLimiter
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -48,6 +58,9 @@ if TYPE_CHECKING:
|
||||
from strix.config.settings import LlmSettings, ReasoningEffort, Settings
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def request_timeout_extra_args(timeout_s: float | None) -> dict[str, float] | None:
|
||||
"""Per-request model timeout; a plain float so ``ModelSettings.to_json_dict()`` stays serializable.""" # noqa: E501
|
||||
if not timeout_s or timeout_s <= 0:
|
||||
@@ -229,6 +242,170 @@ class _NonStreamingModel(Model):
|
||||
yield _completed_stream_event(response, getattr(self._inner, "model", None))
|
||||
|
||||
|
||||
class _TurnGuardModel(Model):
|
||||
"""Keep one turn from corrupting the conversation or running away.
|
||||
|
||||
Tool-call ids: providers that number calls per turn (``exec_command:0``,
|
||||
...) restart the counter each turn, so the same id eventually appears twice
|
||||
in one conversation and strict providers reject every subsequent request.
|
||||
Ids that collide with the history are rewritten before the turn is
|
||||
recorded, and already-corrupted histories are repaired on the way out.
|
||||
|
||||
Tool-call volume: a degenerate response can queue hundreds of calls that
|
||||
the run loop then honours one by one. Only the first
|
||||
``LLM_MAX_TOOL_CALLS_PER_TURN`` calls of a response are kept.
|
||||
|
||||
Stalled streams: a turn that emits a few tokens and then goes silent is
|
||||
not covered by the request timeout, which resets on any byte (keepalives
|
||||
included). ``LLM_STREAM_IDLE_TIMEOUT`` bounds the gap between events so the
|
||||
turn fails instead of hanging, and the existing retry path replays it.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
inner: Model,
|
||||
*,
|
||||
max_tool_calls_per_turn: int = 0,
|
||||
stream_idle_timeout: float = 0.0,
|
||||
) -> None:
|
||||
self._inner = inner
|
||||
self._max_tool_calls_per_turn = max_tool_calls_per_turn
|
||||
self._stream_idle_timeout = stream_idle_timeout
|
||||
|
||||
def _limiter(self) -> TurnToolCallLimiter:
|
||||
return TurnToolCallLimiter(self._max_tool_calls_per_turn)
|
||||
|
||||
def _log_dropped(self, limiter: TurnToolCallLimiter) -> None:
|
||||
if limiter.dropped:
|
||||
logger.warning(
|
||||
"dropped %d tool call(s) past the per-response limit of %d",
|
||||
limiter.dropped,
|
||||
self._max_tool_calls_per_turn,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
await self._inner.close()
|
||||
|
||||
def get_retry_advice(self, request: ModelRetryAdviceRequest) -> ModelRetryAdvice | None:
|
||||
return self._inner.get_retry_advice(request)
|
||||
|
||||
async def get_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem], # noqa: A002
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
*,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: ResponsePromptParam | None,
|
||||
) -> ModelResponse:
|
||||
sanitized = dedupe_input(input)
|
||||
rewriter = TurnCallIdRewriter(sanitized)
|
||||
response = await self._inner.get_response(
|
||||
system_instructions,
|
||||
cast("str | list[TResponseInputItem]", sanitized),
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
tracing,
|
||||
previous_response_id=previous_response_id,
|
||||
conversation_id=conversation_id,
|
||||
prompt=prompt,
|
||||
)
|
||||
limiter = self._limiter()
|
||||
response.output = limiter.filter_items(rewriter.rewrite_items(list(response.output)))
|
||||
self._log_dropped(limiter)
|
||||
return response
|
||||
|
||||
async def stream_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem], # noqa: A002
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
*,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: ResponsePromptParam | None,
|
||||
) -> AsyncIterator[TResponseStreamEvent]:
|
||||
sanitized = dedupe_input(input)
|
||||
rewriter = TurnCallIdRewriter(sanitized)
|
||||
limiter = self._limiter()
|
||||
stream = self._inner.stream_response(
|
||||
system_instructions,
|
||||
cast("str | list[TResponseInputItem]", sanitized),
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
tracing,
|
||||
previous_response_id=previous_response_id,
|
||||
conversation_id=conversation_id,
|
||||
prompt=prompt,
|
||||
)
|
||||
async for event in _with_idle_timeout(stream, self._stream_idle_timeout):
|
||||
guarded = _guard_event(event, rewriter, limiter)
|
||||
if guarded is not None:
|
||||
yield guarded
|
||||
self._log_dropped(limiter)
|
||||
|
||||
|
||||
async def _aclose(stream: AsyncIterator[TResponseStreamEvent]) -> None:
|
||||
if isinstance(stream, AsyncGenerator):
|
||||
with contextlib.suppress(Exception):
|
||||
await stream.aclose()
|
||||
|
||||
|
||||
async def _with_idle_timeout(
|
||||
stream: AsyncIterator[TResponseStreamEvent], timeout: float
|
||||
) -> AsyncIterator[TResponseStreamEvent]:
|
||||
if timeout <= 0:
|
||||
async for event in stream:
|
||||
yield event
|
||||
return
|
||||
|
||||
iterator = stream.__aiter__()
|
||||
while True:
|
||||
try:
|
||||
event = await asyncio.wait_for(iterator.__anext__(), timeout)
|
||||
except StopAsyncIteration:
|
||||
return
|
||||
except TimeoutError:
|
||||
await _aclose(stream)
|
||||
message = f"model stream produced no event for {timeout:.0f}s"
|
||||
logger.warning("%s; abandoning the turn", message)
|
||||
raise TimeoutError(message) from None
|
||||
yield event
|
||||
|
||||
|
||||
def _guard_event(
|
||||
event: TResponseStreamEvent, rewriter: TurnCallIdRewriter, limiter: TurnToolCallLimiter
|
||||
) -> TResponseStreamEvent | None:
|
||||
if isinstance(event, ResponseOutputItemAddedEvent | ResponseOutputItemDoneEvent):
|
||||
rewritten = rewriter.rewrite_item(event.item)
|
||||
if not limiter.allow(rewritten):
|
||||
return None
|
||||
if rewritten is not event.item:
|
||||
return event.model_copy(update={"item": rewritten})
|
||||
return event
|
||||
if isinstance(event, ResponseCompletedEvent):
|
||||
original = list(event.response.output)
|
||||
output = limiter.filter_items(rewriter.rewrite_items(original))
|
||||
if output != original:
|
||||
return event.model_copy(
|
||||
update={"response": event.response.model_copy(update={"output": output})}
|
||||
)
|
||||
return event
|
||||
|
||||
|
||||
def _completed_stream_event(
|
||||
model_response: ModelResponse, model_name: object | None
|
||||
) -> TResponseStreamEvent:
|
||||
@@ -294,19 +471,29 @@ class StrixProvider(MultiProvider):
|
||||
def get_model(self, model_name: str | None) -> Model:
|
||||
llm = load_settings().llm
|
||||
slug = codex.subscription_model(model_name)
|
||||
idle_timeout = float(llm.stream_idle_timeout)
|
||||
if slug:
|
||||
# The ChatGPT subscription backend is always streamed; it has no
|
||||
# non-streaming mode to fall back to, so LLM_DISABLE_STREAMING
|
||||
# does not apply here.
|
||||
return _CodexResponsesModel(
|
||||
model: Model = _CodexResponsesModel(
|
||||
slug,
|
||||
codex.get_subscription_client(),
|
||||
reasoning_effort=llm.reasoning_effort,
|
||||
)
|
||||
model = super().get_model(model_name)
|
||||
if llm.disable_streaming:
|
||||
return _NonStreamingModel(model)
|
||||
return model
|
||||
else:
|
||||
model = super().get_model(model_name)
|
||||
if llm.disable_streaming:
|
||||
model = _NonStreamingModel(model)
|
||||
# The wrapper emits its single event only once the whole request
|
||||
# is done, so an idle gap is meaningless here; the request
|
||||
# timeout bounds it instead.
|
||||
idle_timeout = 0.0
|
||||
return _TurnGuardModel(
|
||||
model,
|
||||
max_tool_calls_per_turn=llm.max_tool_calls_per_turn,
|
||||
stream_idle_timeout=idle_timeout,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_MODEL_RETRY = ModelRetrySettings(
|
||||
|
||||
@@ -57,6 +57,12 @@ class LlmSettings(BaseSettings):
|
||||
alias="LLM_DISABLE_STREAMING",
|
||||
)
|
||||
timeout: int = Field(default=300, alias="LLM_TIMEOUT")
|
||||
stream_idle_timeout: int = Field(default=300, ge=0, alias="LLM_STREAM_IDLE_TIMEOUT")
|
||||
max_tool_calls_per_turn: int = Field(
|
||||
default=32,
|
||||
ge=0,
|
||||
alias="LLM_MAX_TOOL_CALLS_PER_TURN",
|
||||
)
|
||||
|
||||
|
||||
class DedupeSettings(BaseSettings):
|
||||
@@ -122,6 +128,11 @@ class IntegrationSettings(BaseSettings):
|
||||
alias="PERPLEXITY_API_KEY",
|
||||
repr=False,
|
||||
)
|
||||
postman_api_key: str | None = Field(
|
||||
default=None,
|
||||
alias="POSTMAN_API_KEY",
|
||||
repr=False,
|
||||
)
|
||||
|
||||
|
||||
class ViewerSettings(BaseSettings):
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Keep tool-call ids unique within a conversation.
|
||||
|
||||
Some providers return per-turn tool-call ids (``exec_command:0``,
|
||||
``exec_command:1``, ...) whose counter restarts on every turn. Once the same
|
||||
id appears twice in one conversation, the request payload has two assistant
|
||||
tool calls sharing an id and strict providers reject the whole turn, which
|
||||
permanently kills the agent because the malformed history is replayed on
|
||||
every retry. Rewriting duplicates to fresh unique ids keeps the history
|
||||
valid for any provider.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict, deque
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from openai.types.responses import ResponseFunctionToolCall
|
||||
|
||||
|
||||
def new_call_id() -> str:
|
||||
return f"call_{uuid4().hex}"
|
||||
|
||||
|
||||
def collect_call_ids(items: list[Any]) -> set[str]:
|
||||
used: set[str] = set()
|
||||
for item in items:
|
||||
if isinstance(item, dict):
|
||||
call_id = item.get("call_id")
|
||||
if isinstance(call_id, str):
|
||||
used.add(call_id)
|
||||
elif isinstance(item, ResponseFunctionToolCall):
|
||||
used.add(item.call_id)
|
||||
return used
|
||||
|
||||
|
||||
def dedupe_history_call_ids(items: list[Any]) -> tuple[list[Any], bool]:
|
||||
"""Rewrite duplicate call ids in a conversation history.
|
||||
|
||||
Outputs are paired with their call by order, so parallel calls that share
|
||||
an id keep answering the right call after the rewrite.
|
||||
"""
|
||||
used: set[str] = set()
|
||||
pending: dict[str, deque[str]] = defaultdict(deque)
|
||||
rebuilt: list[Any] = []
|
||||
changed = False
|
||||
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
rebuilt.append(item)
|
||||
continue
|
||||
call_id = item.get("call_id")
|
||||
if not isinstance(call_id, str):
|
||||
rebuilt.append(item)
|
||||
continue
|
||||
|
||||
kind = item.get("type")
|
||||
if kind == "function_call":
|
||||
effective = call_id
|
||||
if call_id in used:
|
||||
effective = new_call_id()
|
||||
item = {**item, "call_id": effective} # noqa: PLW2901
|
||||
changed = True
|
||||
used.add(effective)
|
||||
pending[call_id].append(effective)
|
||||
elif kind == "function_call_output":
|
||||
queue = pending.get(call_id)
|
||||
if queue:
|
||||
effective = queue.popleft()
|
||||
if effective != call_id:
|
||||
item = {**item, "call_id": effective} # noqa: PLW2901
|
||||
changed = True
|
||||
rebuilt.append(item)
|
||||
|
||||
return rebuilt, changed
|
||||
|
||||
|
||||
def dedupe_input(model_input: str | list[Any]) -> str | list[Any]:
|
||||
if isinstance(model_input, str):
|
||||
return model_input
|
||||
rebuilt, changed = dedupe_history_call_ids(model_input)
|
||||
return rebuilt if changed else model_input
|
||||
|
||||
|
||||
class TurnCallIdRewriter:
|
||||
"""Rewrite a single turn's tool-call ids that collide with the history.
|
||||
|
||||
A turn's items surface several times (streamed item events, then the
|
||||
completed response), so the same original id must always map to the same
|
||||
replacement within the turn.
|
||||
"""
|
||||
|
||||
def __init__(self, model_input: str | list[Any]) -> None:
|
||||
self._used = set() if isinstance(model_input, str) else collect_call_ids(model_input)
|
||||
self._remap: dict[str, str] = {}
|
||||
self._settled: set[str] = set()
|
||||
|
||||
def rewrite_item(self, item: Any) -> Any:
|
||||
if not isinstance(item, ResponseFunctionToolCall):
|
||||
return item
|
||||
original = item.call_id
|
||||
if original in self._settled:
|
||||
return item
|
||||
replacement = self._remap.get(original)
|
||||
if replacement is None:
|
||||
if original not in self._used:
|
||||
self._used.add(original)
|
||||
self._settled.add(original)
|
||||
return item
|
||||
replacement = new_call_id()
|
||||
self._remap[original] = replacement
|
||||
self._used.add(replacement)
|
||||
self._settled.add(replacement)
|
||||
return item.model_copy(update={"call_id": replacement})
|
||||
|
||||
def rewrite_items(self, items: list[Any]) -> list[Any]:
|
||||
return [self.rewrite_item(item) for item in items]
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Bound how many tool calls one assistant response may queue.
|
||||
|
||||
A degenerate generation can emit hundreds or thousands of tool calls in a
|
||||
single response — typically a poll/wait loop the model writes out ahead of
|
||||
time instead of issuing one call and yielding. The run loop honours all of
|
||||
them, so the agent stops reacting to anything for hours. Keeping only the
|
||||
first ``limit`` calls of a response bounds that blast radius; the model sees
|
||||
their results on the next turn and can reconsider.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from openai.types.responses import ResponseFunctionToolCall
|
||||
|
||||
|
||||
class TurnToolCallLimiter:
|
||||
"""Decide, once per call, whether a turn's tool call is within the limit."""
|
||||
|
||||
def __init__(self, limit: int) -> None:
|
||||
self._limit = limit
|
||||
self._decisions: dict[str, bool] = {}
|
||||
self._kept = 0
|
||||
self.dropped = 0
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self._limit > 0
|
||||
|
||||
def allow(self, item: Any) -> bool:
|
||||
if not self.enabled or not isinstance(item, ResponseFunctionToolCall):
|
||||
return True
|
||||
decided = self._decisions.get(item.call_id)
|
||||
if decided is not None:
|
||||
return decided
|
||||
allowed = self._kept < self._limit
|
||||
if allowed:
|
||||
self._kept += 1
|
||||
else:
|
||||
self.dropped += 1
|
||||
self._decisions[item.call_id] = allowed
|
||||
return allowed
|
||||
|
||||
def filter_items(self, items: list[Any]) -> list[Any]:
|
||||
return [item for item in items if self.allow(item)]
|
||||
@@ -780,17 +780,21 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
await coordinator.set_status(agent_id, "failed", error=str(exc))
|
||||
await notify_parent_on_terminal(coordinator, agent_id, "failed")
|
||||
return None
|
||||
if not interactive:
|
||||
raise
|
||||
if isinstance(exc, MaxTurnsExceeded):
|
||||
status: Status = "stopped"
|
||||
elif isinstance(exc, UserError | AgentsException | APIError):
|
||||
status = "failed"
|
||||
else:
|
||||
status = "crashed"
|
||||
logger.exception("agent run failed for %s; parking as %s", agent_id, status)
|
||||
logger.exception("agent run failed for %s; marking %s", agent_id, status)
|
||||
# Settle the status and wake the parent before the exception unwinds a
|
||||
# non-interactive agent's task: a child that dies still owes its parent a
|
||||
# report, and the parent would otherwise wait out its timeout on a message
|
||||
# the dead child can no longer send.
|
||||
await coordinator.set_status(agent_id, status, error=str(exc) or type(exc).__name__)
|
||||
await notify_parent_on_terminal(coordinator, agent_id, status)
|
||||
if not interactive:
|
||||
raise
|
||||
return None
|
||||
else:
|
||||
return cast("RunResultBase | None", stream)
|
||||
@@ -834,7 +838,7 @@ async def _append_tool_required_message(
|
||||
)
|
||||
else:
|
||||
message = (
|
||||
"Your previous response ended the autonomous Strix run without a lifecycle tool "
|
||||
"Your previous response ended the autonomous run without a lifecycle tool "
|
||||
"call. That is invalid in non-interactive mode; plain text final answers are "
|
||||
"ignored. Continue immediately and call exactly one tool. "
|
||||
f"If your work is complete, call {finish_tool}. "
|
||||
|
||||
@@ -20,6 +20,8 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
LLM_TURN_KEY = "llm_turn"
|
||||
|
||||
_STAGE_LABELS: tuple[str, ...] = ("NOTICE", "URGENT", "CRITICAL")
|
||||
_TURN_WARN_BANDS: tuple[float, ...] = (0.70, 0.85, 0.95)
|
||||
_ROOT_BUDGET_WARN_BANDS: tuple[float, ...] = (0.70, 0.85, 0.95)
|
||||
@@ -144,6 +146,7 @@ class ReportUsageHooks(RunHooks[dict[str, Any]]):
|
||||
system_prompt: str | None, # noqa: ARG002
|
||||
input_items: list[TResponseInputItem],
|
||||
) -> None:
|
||||
context.context[LLM_TURN_KEY] = int(context.context.get(LLM_TURN_KEY, 0)) + 1
|
||||
try:
|
||||
self._maybe_warn_turns(context, input_items)
|
||||
self._maybe_warn_budget(context, input_items)
|
||||
|
||||
+57
-15
@@ -33,6 +33,50 @@ def _accepts_required_tool_choice(model_name: str | None) -> bool:
|
||||
return name.startswith("openai/") or is_known_openai_bare_model(name)
|
||||
|
||||
|
||||
def _render_diff_scope(diff_scope: dict[str, Any]) -> list[str]:
|
||||
"""Render pull-request diff-scope constraints as root-task lines."""
|
||||
if not diff_scope.get("active"):
|
||||
return []
|
||||
parts: list[str] = [
|
||||
"\n\nScope Constraints:",
|
||||
"- Pull request diff-scope mode is active. Prioritize changed files "
|
||||
"and use other files only for context.",
|
||||
]
|
||||
for repo_scope in diff_scope.get("repos", []) or []:
|
||||
label = repo_scope.get("workspace_subdir") or repo_scope.get("source_path") or "repository"
|
||||
changed = repo_scope.get("analyzable_files_count", 0)
|
||||
deleted = repo_scope.get("deleted_files_count", 0)
|
||||
parts.append(f"- {label}: {changed} changed file(s) in primary scope")
|
||||
if deleted:
|
||||
parts.append(f"- {label}: {deleted} deleted file(s) are context-only")
|
||||
return parts
|
||||
|
||||
|
||||
def _render_api_spec(details: dict[str, Any]) -> list[str]:
|
||||
"""Render an API spec target as root-task lines.
|
||||
|
||||
The spec itself is in the workspace, so the task points at the file and lets
|
||||
the agent read the contract rather than restating a parsed summary of it.
|
||||
"""
|
||||
title = details.get("spec_title") or details.get("target_spec", "API")
|
||||
workspace_path = details.get("workspace_path", "")
|
||||
lines = [
|
||||
f"- {title} ({details.get('spec_format', 'api')} specification"
|
||||
+ (f", available at: {workspace_path}" if workspace_path else "")
|
||||
+ ")"
|
||||
]
|
||||
if base_urls := details.get("base_urls") or []:
|
||||
lines.append(" - Base URL(s): " + ", ".join(base_urls))
|
||||
lines.append(
|
||||
" - Read the specification and test every operation it declares, using "
|
||||
"its declared parameters, request bodies, and auth. Endpoints in the "
|
||||
"specification are in scope even when nothing links to them. Load the "
|
||||
"`api_spec_testing` skill for the methodology, or spawn a specialist "
|
||||
"with it."
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def build_root_task(scan_config: dict[str, Any]) -> str:
|
||||
targets = scan_config.get("targets", []) or []
|
||||
diff_scope = scan_config.get("diff_scope") or {}
|
||||
@@ -43,6 +87,7 @@ def build_root_task(scan_config: dict[str, Any]) -> str:
|
||||
"Local Codebases": [],
|
||||
"URLs": [],
|
||||
"IP Addresses": [],
|
||||
"API Specifications": [],
|
||||
}
|
||||
|
||||
for target in targets:
|
||||
@@ -68,6 +113,8 @@ def build_root_task(scan_config: dict[str, Any]) -> str:
|
||||
sections["URLs"].append(f"- {details.get('target_url', '')}")
|
||||
elif ttype == "ip_address":
|
||||
sections["IP Addresses"].append(f"- {details.get('target_ip', '')}")
|
||||
elif ttype == "api_spec":
|
||||
sections["API Specifications"].extend(_render_api_spec(details))
|
||||
|
||||
parts: list[str] = []
|
||||
for label, items in sections.items():
|
||||
@@ -92,21 +139,7 @@ def build_root_task(scan_config: dict[str, Any]) -> str:
|
||||
"truth for what to do."
|
||||
)
|
||||
|
||||
if diff_scope.get("active"):
|
||||
parts.append("\n\nScope Constraints:")
|
||||
parts.append(
|
||||
"- Pull request diff-scope mode is active. Prioritize changed files "
|
||||
"and use other files only for context.",
|
||||
)
|
||||
for repo_scope in diff_scope.get("repos", []) or []:
|
||||
label = (
|
||||
repo_scope.get("workspace_subdir") or repo_scope.get("source_path") or "repository"
|
||||
)
|
||||
changed = repo_scope.get("analyzable_files_count", 0)
|
||||
deleted = repo_scope.get("deleted_files_count", 0)
|
||||
parts.append(f"- {label}: {changed} changed file(s) in primary scope")
|
||||
if deleted:
|
||||
parts.append(f"- {label}: {deleted} deleted file(s) are context-only")
|
||||
parts.extend(_render_diff_scope(diff_scope))
|
||||
|
||||
task = " ".join(parts)
|
||||
if user_instructions:
|
||||
@@ -121,6 +154,7 @@ def build_scope_context(scan_config: dict[str, Any]) -> dict[str, Any]:
|
||||
"local_code": "target_path",
|
||||
"web_application": "target_url",
|
||||
"ip_address": "target_ip",
|
||||
"api_spec": "target_spec",
|
||||
}
|
||||
for target in scan_config.get("targets", []) or []:
|
||||
ttype = target.get("type", "unknown")
|
||||
@@ -134,6 +168,14 @@ def build_scope_context(scan_config: dict[str, Any]) -> dict[str, Any]:
|
||||
{"type": ttype, "value": value, "workspace_path": workspace_path},
|
||||
)
|
||||
|
||||
# An API spec authorizes the hosts it declares as in-scope web targets
|
||||
# so the agent can exercise every endpoint without expanding scope.
|
||||
if ttype == "api_spec":
|
||||
authorized.extend(
|
||||
{"type": "web_application", "value": base_url, "workspace_path": ""}
|
||||
for base_url in details.get("base_urls") or []
|
||||
)
|
||||
|
||||
return {
|
||||
"scope_source": "system_scan_config",
|
||||
"authorization_source": "strix_platform_verified_targets",
|
||||
|
||||
@@ -268,6 +268,9 @@ async def run_strix_scan(
|
||||
model_settings=model_settings,
|
||||
sandbox=SandboxRunConfig(client=bundle["client"], session=bundle["session"]),
|
||||
trace_include_sensitive_data=False,
|
||||
# A hallucinated tool name is a recoverable model mistake, not a scan-ending
|
||||
# error: hand it back as a tool result so the agent can correct itself.
|
||||
tool_not_found_behavior="return_error_to_model",
|
||||
)
|
||||
hooks = ReportUsageHooks(
|
||||
model=resolved_model,
|
||||
@@ -290,7 +293,7 @@ async def run_strix_scan(
|
||||
)
|
||||
|
||||
root_agent = build_strix_agent(
|
||||
name="Strix",
|
||||
name="Root Agent",
|
||||
skills=skills,
|
||||
is_root=True,
|
||||
scan_mode=scan_mode,
|
||||
@@ -304,7 +307,7 @@ async def run_strix_scan(
|
||||
if not is_resume:
|
||||
await coordinator.register(
|
||||
root_id,
|
||||
"Strix",
|
||||
"Root Agent",
|
||||
parent_id=None,
|
||||
task=root_task,
|
||||
skills=skills,
|
||||
|
||||
@@ -65,6 +65,14 @@ Examples:
|
||||
# Local code analysis
|
||||
strix --target ./my-project
|
||||
|
||||
# API spec test (OpenAPI/Swagger file or Postman collection export)
|
||||
strix --target ./openapi.yaml --target https://api.example.com
|
||||
strix --target ./collection.postman_collection.json
|
||||
|
||||
# Postman collection pulled live by id (needs POSTMAN_API_KEY); optional environment
|
||||
strix --target postman://<collection-uuid> --target https://api.example.com
|
||||
strix --target "postman://<collection-uuid>?env=<environment-uuid>"
|
||||
|
||||
# Domain penetration test
|
||||
strix --target example.com
|
||||
|
||||
@@ -107,8 +115,10 @@ Examples:
|
||||
"--target",
|
||||
type=str,
|
||||
action="append",
|
||||
help="Target to test (URL, repository, local directory path, domain name, or IP address). "
|
||||
"Local directories are mounted into the sandbox writable. "
|
||||
help="Target to test: URL, repository, local directory path, domain name, IP address, "
|
||||
"an API spec file (OpenAPI/Swagger .json/.yaml or a Postman collection export), or a "
|
||||
"Postman collection by id (postman://<collection-uuid>[?env=<environment-uuid>], needs "
|
||||
"POSTMAN_API_KEY). Local directories are mounted into the sandbox writable. "
|
||||
"Can be specified multiple times for multi-target scans. "
|
||||
"Fresh runs require --target or --target-list.",
|
||||
)
|
||||
|
||||
@@ -12,7 +12,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from strix.config import Settings, codex, load_settings
|
||||
from strix.core.paths import run_dir_for
|
||||
@@ -28,8 +28,18 @@ from strix.interface.utils import (
|
||||
read_target_list_file,
|
||||
resolve_diff_scope_context,
|
||||
rewrite_localhost_targets,
|
||||
stage_api_specs,
|
||||
write_fetched_collection,
|
||||
)
|
||||
from strix.telemetry import posthog, scarf
|
||||
from strix.utils.api_spec import (
|
||||
SpecParseError,
|
||||
fetch_postman_collection,
|
||||
fetch_postman_environment,
|
||||
load_spec,
|
||||
spec_base_urls,
|
||||
spec_title,
|
||||
)
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -109,6 +119,9 @@ def build_targets_info(args: argparse.Namespace) -> None:
|
||||
else:
|
||||
display_target = target
|
||||
|
||||
if target_type == "api_spec":
|
||||
_resolve_api_spec(target, target_dict)
|
||||
|
||||
args.targets_info.append(
|
||||
{"type": target_type, "details": target_dict, "original": display_target}
|
||||
)
|
||||
@@ -119,6 +132,34 @@ def build_targets_info(args: argparse.Namespace) -> None:
|
||||
rewrite_localhost_targets(args.targets_info, HOST_GATEWAY_HOSTNAME)
|
||||
|
||||
|
||||
def _resolve_api_spec(target: str, details: dict[str, Any]) -> None:
|
||||
"""Read the spec up front so bad input fails before the run starts.
|
||||
|
||||
Records the declared base URLs (the only thing scope authorization can take
|
||||
from a spec) and, for a ``postman://`` target, downloads the collection to a
|
||||
local file so the sandbox never needs the Postman API key.
|
||||
"""
|
||||
try:
|
||||
if details.get("source") == "postman_api":
|
||||
collection_uid = str(details["collection_uid"])
|
||||
api_key = load_settings().integrations.postman_api_key or ""
|
||||
raw = fetch_postman_collection(collection_uid, api_key)
|
||||
environment_uid = str(details.get("environment_uid") or "")
|
||||
extra_variables = (
|
||||
fetch_postman_environment(environment_uid, api_key) if environment_uid else None
|
||||
)
|
||||
details["target_spec"] = write_fetched_collection(raw, collection_uid)
|
||||
else:
|
||||
raw = load_spec(str(details["target_spec"]))
|
||||
extra_variables = None
|
||||
base_urls = spec_base_urls(raw, extra_variables=extra_variables)
|
||||
except SpecParseError as exc:
|
||||
raise ValueError(f"Invalid API spec '{target}': {exc}") from None
|
||||
|
||||
details["spec_title"] = spec_title(raw)
|
||||
details["base_urls"] = base_urls
|
||||
|
||||
|
||||
def prepare_run(args: argparse.Namespace) -> None:
|
||||
"""Resolve the run name, clone repos, compute diff-scope, and persist state.
|
||||
|
||||
@@ -139,6 +180,7 @@ def prepare_run(args: argparse.Namespace) -> None:
|
||||
target_info["details"]["cloned_repo_path"] = cloned_path
|
||||
|
||||
args.local_sources = collect_local_sources(args.targets_info)
|
||||
args.local_sources.extend(stage_api_specs(args.targets_info, args.run_name))
|
||||
diff_scope = resolve_diff_scope_context(
|
||||
local_sources=args.local_sources,
|
||||
scope_mode=args.scope_mode,
|
||||
|
||||
@@ -1169,3 +1169,120 @@ func TestChatContentRerendersOnWidthAndExpansionChange(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A model or backend failure can be a wrapped exception hundreds of columns
|
||||
// wide and several lines long. The status row is one line of the chat column, so
|
||||
// an oversized one widens the whole column - JoinHorizontal pads every row to the
|
||||
// widest - which pushed the sidebar off screen and wrapped the frame.
|
||||
func TestLongErrorDoesNotBreakTheFrame(t *testing.T) {
|
||||
model := New(nil)
|
||||
model.width, model.height = 120, 24
|
||||
model.showSplash = false
|
||||
model.handleEnvelope(stateEnvelope(t, 1, protocol.Snapshot{ScanState: "running"}))
|
||||
bootstrap := protocol.CollectionBootstrap{
|
||||
Collection: "agents", Revision: 1, Cursor: 0, NextCursor: 1, Done: true,
|
||||
Items: []json.RawMessage{rawJSON(t, protocol.Agent{ID: "a0", Name: "Strix", Status: "running"})},
|
||||
}
|
||||
model.handleEnvelope(protocol.Envelope{
|
||||
Version: protocol.Version, Type: "collection_bootstrap", Payload: rawJSON(t, bootstrap),
|
||||
})
|
||||
model.errorText = "litellm.APIConnectionError: OpenrouterException - Connection error " +
|
||||
"while calling https://openrouter.ai/api/v1/chat/completions: HTTPSConnectionPool" +
|
||||
"(host='openrouter.ai', port=443): Max retries exceeded\nTraceback (most recent " +
|
||||
"call last):\n File \"/x/y.py\", line 42, in send\n raise err"
|
||||
model.resizeViewport()
|
||||
|
||||
lines := strings.Split(model.View(), "\n")
|
||||
if len(lines) > model.height {
|
||||
t.Fatalf("frame is %d rows in a %d-row terminal", len(lines), model.height)
|
||||
}
|
||||
for i, line := range lines {
|
||||
if width := ansi.StringWidth(line); width > model.width {
|
||||
t.Fatalf("row %d is %d columns in a %d-column terminal", i, width, model.width)
|
||||
}
|
||||
}
|
||||
// The sidebar has to survive: its panels are the right edge of the frame.
|
||||
if !strings.Contains(ansi.Strip(model.View()), "Strix") {
|
||||
t.Fatal("the agent tree was pushed out of the frame")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusMessageFlattensAndKeepsItsHint(t *testing.T) {
|
||||
row := ansi.Strip(statusMessage("boom\nsecond line\twith tabs", red, " · Send message to resume", 60))
|
||||
|
||||
if strings.Contains(row, "\n") || strings.Contains(row, "\t") {
|
||||
t.Fatalf("status row is not a single line: %q", row)
|
||||
}
|
||||
if !strings.HasSuffix(row, " · Send message to resume") {
|
||||
t.Fatalf("the hint was lost: %q", row)
|
||||
}
|
||||
if !strings.Contains(row, "boom second line with tabs") {
|
||||
t.Fatalf("the message was mangled: %q", row)
|
||||
}
|
||||
// A message far too long for the row keeps the hint readable.
|
||||
long := ansi.Strip(statusMessage(strings.Repeat("x", 500), red, " · Send message to resume", 60))
|
||||
if width := ansi.StringWidth(long); width > 60 {
|
||||
t.Fatalf("status message is %d columns, want at most 60", width)
|
||||
}
|
||||
if !strings.HasSuffix(long, " · Send message to resume") {
|
||||
t.Fatalf("the hint was clipped away: %q", long)
|
||||
}
|
||||
}
|
||||
|
||||
// The status row must be exactly as wide as the column it sits in, at every
|
||||
// terminal size. A narrow terminal cannot fit the quit hint alongside any status
|
||||
// text, and keeping it anyway made the row wider than the terminal.
|
||||
func TestStatusRowIsExactlyItsWidth(t *testing.T) {
|
||||
quitHint := lipgloss.NewStyle().Foreground(white).Render("ctrl-q") +
|
||||
lipgloss.NewStyle().Foreground(dim).Render(" quit")
|
||||
longMessage := lipgloss.NewStyle().Foreground(red).Render(strings.Repeat("boom ", 40))
|
||||
|
||||
for width := 1; width <= 60; width++ {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
left, right string
|
||||
}{
|
||||
{"empty", "", ""},
|
||||
{"hint only", "", quitHint},
|
||||
{"long message and hint", longMessage, quitHint},
|
||||
{"long message alone", longMessage, ""},
|
||||
} {
|
||||
row := composeStatusRow(testCase.left, testCase.right, width)
|
||||
if got := ansi.StringWidth(row); got != width {
|
||||
t.Fatalf("%s at width %d rendered %d columns: %q",
|
||||
testCase.name, width, got, ansi.Strip(row))
|
||||
}
|
||||
if strings.Contains(row, "\n") {
|
||||
t.Fatalf("%s at width %d spans rows", testCase.name, width)
|
||||
}
|
||||
}
|
||||
}
|
||||
if row := composeStatusRow("x", "y", 0); row != "" {
|
||||
t.Fatalf("a zero-width row should be empty, got %q", row)
|
||||
}
|
||||
}
|
||||
|
||||
// A running scan in a narrow terminal must not wrap the frame.
|
||||
func TestNarrowTerminalKeepsTheFrameIntact(t *testing.T) {
|
||||
for _, width := range []int{8, 10, 13, 14, 20, 40} {
|
||||
model := New(nil)
|
||||
model.width, model.height = width, 20
|
||||
model.showSplash = false
|
||||
model.handleEnvelope(stateEnvelope(t, 1, protocol.Snapshot{ScanState: "running"}))
|
||||
bootstrap := protocol.CollectionBootstrap{
|
||||
Collection: "agents", Revision: 1, Cursor: 0, NextCursor: 1, Done: true,
|
||||
Items: []json.RawMessage{rawJSON(t, protocol.Agent{ID: "a0", Name: "Strix", Status: "running"})},
|
||||
}
|
||||
model.handleEnvelope(protocol.Envelope{
|
||||
Version: protocol.Version, Type: "collection_bootstrap", Payload: rawJSON(t, bootstrap),
|
||||
})
|
||||
model.errorText = strings.Repeat("connection failed ", 20)
|
||||
model.resizeViewport()
|
||||
|
||||
for i, line := range strings.Split(model.View(), "\n") {
|
||||
if got := ansi.StringWidth(line); got > width {
|
||||
t.Fatalf("at width %d row %d is %d columns", width, i, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,8 +214,11 @@ func (m *Model) setupLogAppend(line string) {
|
||||
}
|
||||
|
||||
// setupMsg appends a styled feedback line (success green, error red, notice dim).
|
||||
// The log budgets rows by entry, so a message is flattened to one line first: a
|
||||
// wrapped exception would otherwise render as several rows and push the launch
|
||||
// column past the bottom of the terminal.
|
||||
func (m *Model) setupMsg(text string, style lipgloss.Style) {
|
||||
m.setupLogAppend(style.Render(text))
|
||||
m.setupLogAppend(style.Render(flattenStatus(text)))
|
||||
}
|
||||
|
||||
// setupLogRows is how many feedback lines the launch column shows before the
|
||||
|
||||
@@ -93,3 +93,25 @@ func TestFocusedPanelsCarryTheGreenBorder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A wrapped exception is several lines. The log budgets rows by entry, so it has
|
||||
// to become one row or the launch column grows past the terminal.
|
||||
func TestSetupLogKeepsMultiLineErrorsToOneRow(t *testing.T) {
|
||||
model := New(nil)
|
||||
model.width, model.height = 100, 26
|
||||
model.showSplash = false
|
||||
model.handleEnvelope(stateEnvelope(t, 1, protocol.Snapshot{SetupMode: true, ScanState: "setup"}))
|
||||
model.setupMsg("boom\nTraceback (most recent call last):\n File \"x.py\", line 1\n raise", render.Col(red))
|
||||
model.resizeViewport()
|
||||
|
||||
if entries := len(model.setupLog); entries != 1 {
|
||||
t.Fatalf("one message became %d log entries", entries)
|
||||
}
|
||||
if strings.Contains(model.setupLog[0], "\n") {
|
||||
t.Fatalf("log entry spans rows: %q", model.setupLog[0])
|
||||
}
|
||||
lines := strings.Split(model.View(), "\n")
|
||||
if len(lines) > model.height {
|
||||
t.Fatalf("start screen is %d rows in a %d-row terminal", len(lines), model.height)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -654,8 +654,7 @@ func (m Model) statusView(width int) string {
|
||||
case "waiting":
|
||||
left = lipgloss.NewStyle().Foreground(dim).Render("Send message to resume")
|
||||
if msg := agent.ErrorMessage; msg != "" {
|
||||
left = lipgloss.NewStyle().Foreground(red).Render(msg) +
|
||||
lipgloss.NewStyle().Foreground(dim).Render(" · Send message to resume")
|
||||
left = statusMessage(msg, red, " · Send message to resume", width)
|
||||
}
|
||||
case "budget_paused":
|
||||
left = lipgloss.NewStyle().Foreground(amber).Render("Budget limit reached") +
|
||||
@@ -670,15 +669,54 @@ func (m Model) statusView(width int) string {
|
||||
if msg == "" {
|
||||
msg = "Agent failed"
|
||||
}
|
||||
left = lipgloss.NewStyle().Foreground(red).Render(msg) +
|
||||
lipgloss.NewStyle().Foreground(dim).Render(" · Send message to resume")
|
||||
left = statusMessage(msg, red, " · Send message to resume", width)
|
||||
}
|
||||
}
|
||||
if m.errorText != "" {
|
||||
left = lipgloss.NewStyle().Foreground(red).Render(m.errorText)
|
||||
left = statusMessage(m.errorText, red, "", width-lipgloss.Width(right))
|
||||
}
|
||||
gap := max(1, width-lipgloss.Width(left)-lipgloss.Width(right))
|
||||
return " " + left + strings.Repeat(" ", max(1, gap-1)) + right
|
||||
return composeStatusRow(left, right, width)
|
||||
}
|
||||
|
||||
// composeStatusRow lays the status text and the corner hint on one row exactly
|
||||
// width columns wide. A wider row would widen the whole chat column, because
|
||||
// JoinHorizontal pads every row of a block to its widest, which pushes the
|
||||
// sidebar off screen and wraps the frame.
|
||||
func composeStatusRow(left, right string, width int) string {
|
||||
if width <= 0 {
|
||||
return ""
|
||||
}
|
||||
const leading = 1 // the row is indented one column, like the panels above it
|
||||
// A terminal can be narrower than the hint itself. Drop the hint rather than
|
||||
// keep it at the cost of the status, which is the part carrying information;
|
||||
// ctrl-q works whether or not the row has room to say so.
|
||||
if lipgloss.Width(right) > 0 && width < lipgloss.Width(right)+leading+2 {
|
||||
right = ""
|
||||
}
|
||||
separator := 0
|
||||
if lipgloss.Width(right) > 0 {
|
||||
separator = 1
|
||||
}
|
||||
left = truncate(left, max(0, width-leading-lipgloss.Width(right)-separator))
|
||||
padding := max(0, width-leading-lipgloss.Width(left)-lipgloss.Width(right))
|
||||
return " " + left + strings.Repeat(" ", padding) + right
|
||||
}
|
||||
|
||||
// statusMessage fits a message and its trailing hint on the one status row. A
|
||||
// model or backend error can be a wrapped exception several lines long, so it is
|
||||
// flattened to a single line and clipped, leaving the hint readable.
|
||||
func statusMessage(message string, color lipgloss.Color, hint string, width int) string {
|
||||
styledHint := lipgloss.NewStyle().Foreground(dim).Render(hint)
|
||||
room := max(1, width-2-lipgloss.Width(styledHint))
|
||||
flat := truncate(flattenStatus(message), room)
|
||||
return lipgloss.NewStyle().Foreground(color).Render(flat) + styledHint
|
||||
}
|
||||
|
||||
// flattenStatus turns a multi-line message into one line, collapsing the runs of
|
||||
// whitespace that joining its lines leaves behind.
|
||||
func flattenStatus(message string) string {
|
||||
message = strings.NewReplacer("\r\n", " ", "\r", " ", "\n", " ", "\t", " ").Replace(message)
|
||||
return strings.Join(strings.Fields(message), " ")
|
||||
}
|
||||
|
||||
func (m Model) sweepView() string {
|
||||
|
||||
@@ -74,6 +74,8 @@ func vulnerabilityMarkdownReport(v map[string]any) string {
|
||||
field("Ecosystem", render.StringValue(dep["package_ecosystem"]))
|
||||
field("Installed Version", render.StringValue(dep["installed_version"]))
|
||||
field("Fixed Version", render.StringValue(dep["fixed_version"]))
|
||||
field("Introduced By", render.StringValue(dep["introduced_by"]))
|
||||
field("Dependency Chain", render.StringValue(dep["dependency_path"]))
|
||||
}
|
||||
field("Endpoint", render.StringValue(v["endpoint"]))
|
||||
field("Method", render.StringValue(v["method"]))
|
||||
|
||||
@@ -298,6 +298,8 @@ func vulnerabilityBody(v map[string]any) string {
|
||||
field("Ecosystem", render.StringValue(dep["package_ecosystem"]))
|
||||
field("Installed Version", render.StringValue(dep["installed_version"]))
|
||||
field("Fixed Version", render.StringValue(dep["fixed_version"]))
|
||||
field("Introduced By", render.StringValue(dep["introduced_by"]))
|
||||
field("Dependency Chain", render.StringValue(dep["dependency_path"]))
|
||||
}
|
||||
field("Endpoint", render.StringValue(v["endpoint"]))
|
||||
field("Method", render.StringValue(v["method"]))
|
||||
|
||||
@@ -57,6 +57,12 @@ func renderDependencyReport(args map[string]any, result any) string {
|
||||
section("Description", StringValue(args["description"]))
|
||||
section("Impact", StringValue(args["impact"]))
|
||||
section("Technical Analysis", StringValue(args["technical_analysis"]))
|
||||
if reach := StringValue(args["reachability"]); reach != "" && reach != "unknown" {
|
||||
b.WriteString("\n\n" + Bold(Field).Render("Usage evidence: ") + reach)
|
||||
if ev := StringValue(args["reachability_evidence"]); ev != "" {
|
||||
b.WriteString("\n" + ev)
|
||||
}
|
||||
}
|
||||
section("Assumptions", StringValue(args["assumptions"]))
|
||||
section("Remediation", StringValue(args["remediation_steps"]))
|
||||
if title == "" {
|
||||
|
||||
@@ -431,7 +431,7 @@ _INTERNAL_TURN_PREFIXES = (
|
||||
"== Inherited context from parent",
|
||||
# strix.core.execution: the no-tool-call recovery nudge, both modes.
|
||||
"Your previous message ended a turn without a tool call.",
|
||||
"Your previous response ended the autonomous Strix run without a lifecycle tool call.",
|
||||
"Your previous response ended the autonomous run without a lifecycle tool call.",
|
||||
# strix.core.hooks: budget warnings, the only notices injected unwrapped.
|
||||
*(
|
||||
f"[{label}] {subject}"
|
||||
|
||||
@@ -11,7 +11,7 @@ import tempfile
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import docker
|
||||
import requests
|
||||
@@ -21,6 +21,7 @@ from rich.panel import Panel
|
||||
from rich.text import Text
|
||||
|
||||
from strix.config import load_settings
|
||||
from strix.utils.api_spec import detect_spec_format
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -484,6 +485,15 @@ def _derive_target_label_for_run_name(targets_info: list[dict[str, Any]] | None)
|
||||
if target_type == "ip_address":
|
||||
return str(details.get("target_ip", original) or original)
|
||||
|
||||
if target_type == "api_spec":
|
||||
if details.get("source") == "postman_api":
|
||||
return "postman-collection"
|
||||
spec_path = details.get("target_spec", original)
|
||||
try:
|
||||
return str(Path(spec_path).stem or spec_path)
|
||||
except Exception:
|
||||
return str(spec_path)
|
||||
|
||||
return str(original or "pentest")
|
||||
|
||||
|
||||
@@ -1092,7 +1102,7 @@ def resolve_diff_scope_context(
|
||||
def _is_http_git_repo(url: str) -> bool:
|
||||
check_url = f"{url.rstrip('/')}/info/refs?service=git-upload-pack"
|
||||
try:
|
||||
with requests.get(check_url, headers={"User-Agent": "git/strix"}, timeout=10) as resp:
|
||||
with requests.get(check_url, headers={"User-Agent": "git/2.43.0"}, timeout=10) as resp:
|
||||
if resp.status_code >= 400:
|
||||
return resp.status_code == 401
|
||||
return "x-git-upload-pack-advertisement" in resp.headers.get("Content-Type", "")
|
||||
@@ -1113,6 +1123,24 @@ def infer_target_type(target: str) -> tuple[str, dict[str, str]]: # noqa: PLR09
|
||||
return "repository", {"target_repo": target}
|
||||
|
||||
parsed = urlparse(target)
|
||||
if parsed.scheme == "postman":
|
||||
collection_uid = f"{parsed.netloc}{parsed.path}".strip("/")
|
||||
if not collection_uid:
|
||||
raise ValueError(
|
||||
f"Missing Postman collection id in '{target}' (expected postman://<collection-uid>)"
|
||||
)
|
||||
details = {
|
||||
"target_spec": target,
|
||||
"spec_format": "postman",
|
||||
"source": "postman_api",
|
||||
"collection_uid": collection_uid,
|
||||
}
|
||||
query = parse_qs(parsed.query)
|
||||
env_uid = (query.get("env") or query.get("environment") or [""])[0].strip()
|
||||
if env_uid:
|
||||
details["environment_uid"] = env_uid
|
||||
return "api_spec", details
|
||||
|
||||
if parsed.scheme in ("http", "https"):
|
||||
if parsed.username or parsed.password:
|
||||
return "repository", {"target_repo": target}
|
||||
@@ -1138,6 +1166,12 @@ def infer_target_type(target: str) -> tuple[str, dict[str, str]]: # noqa: PLR09
|
||||
if path.is_dir():
|
||||
check_mountable_dir(path)
|
||||
return "local_code", {"target_path": str(path.resolve())}
|
||||
spec_format = detect_spec_format(path)
|
||||
if spec_format is not None:
|
||||
return "api_spec", {
|
||||
"target_spec": str(path.resolve()),
|
||||
"spec_format": spec_format,
|
||||
}
|
||||
raise ValueError(f"Path exists but is not a directory: {target}")
|
||||
except (OSError, RuntimeError) as e:
|
||||
raise ValueError(f"Invalid path: {target} - {e!s}") from e
|
||||
@@ -1164,6 +1198,9 @@ def infer_target_type(target: str) -> tuple[str, dict[str, str]]: # noqa: PLR09
|
||||
"- A valid URL (http:// or https://)\n"
|
||||
"- A Git repository URL (https://host/org/repo or git@host:org/repo.git)\n"
|
||||
"- A local directory path\n"
|
||||
"- An API spec file (OpenAPI/Swagger .json/.yaml or a Postman collection)\n"
|
||||
"- A Postman collection by id (postman://<collection-uid>[?env=<environment-uid>], "
|
||||
"needs POSTMAN_API_KEY)\n"
|
||||
"- A domain name (e.g., example.com)\n"
|
||||
"- An IP address (e.g., 192.168.1.10)"
|
||||
)
|
||||
@@ -1438,6 +1475,62 @@ def rewrite_localhost_targets(targets_info: list[dict[str, Any]], host_gateway:
|
||||
details["target_ip"] = host_gateway
|
||||
|
||||
|
||||
#: API spec targets are copied into one workspace directory rather than mounted
|
||||
#: from wherever they happen to live on the host.
|
||||
API_SPEC_WORKSPACE_SUBDIR = "api-specs"
|
||||
|
||||
|
||||
def write_fetched_collection(collection: dict[str, Any], collection_uid: str) -> str:
|
||||
"""Write a collection fetched from the Postman API to a local file.
|
||||
|
||||
Returns the file path, so a ``postman://`` target continues as an ordinary
|
||||
spec file from here on and the API key never leaves the host.
|
||||
"""
|
||||
staging = Path(tempfile.gettempdir()) / "strix_api_specs" / "fetched"
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
path = staging / f"{sanitize_name(collection_uid)}.postman_collection.json"
|
||||
path.write_text(json.dumps(collection, indent=2), encoding="utf-8")
|
||||
return str(path)
|
||||
|
||||
|
||||
def stage_api_specs(targets_info: list[dict[str, Any]], run_name: str) -> list[dict[str, Any]]:
|
||||
"""Copy every ``api_spec`` target into one directory for the sandbox.
|
||||
|
||||
A spec is a single file the agent reads, not a tree it works in, so it is
|
||||
copied to a per-run staging directory that is exposed at
|
||||
``/workspace/api-specs`` instead of mounting its host location. Each target's
|
||||
``workspace_path`` records where the agent will find it.
|
||||
"""
|
||||
specs = [t for t in targets_info if t.get("type") == "api_spec"]
|
||||
if not specs:
|
||||
return []
|
||||
|
||||
staging = Path(tempfile.gettempdir()) / "strix_api_specs" / run_name
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
used: set[str] = set()
|
||||
for target in specs:
|
||||
details = target["details"]
|
||||
source = Path(str(details["target_spec"]))
|
||||
name = source.name
|
||||
stem, suffix = source.stem, source.suffix
|
||||
count = 1
|
||||
while name in used:
|
||||
count += 1
|
||||
name = f"{stem}-{count}{suffix}"
|
||||
used.add(name)
|
||||
shutil.copy2(source, staging / name)
|
||||
details["workspace_path"] = f"/workspace/{API_SPEC_WORKSPACE_SUBDIR}/{name}"
|
||||
|
||||
return [
|
||||
{
|
||||
"source_path": str(staging),
|
||||
"workspace_subdir": API_SPEC_WORKSPACE_SUBDIR,
|
||||
"protect_metadata": False,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def clone_repository(repo_url: str, run_name: str, dest_name: str | None = None) -> str:
|
||||
console = Console()
|
||||
|
||||
|
||||
@@ -183,6 +183,24 @@ def _dependency_identity(report: dict[str, Any]) -> tuple[str, str, str] | None:
|
||||
return cve, ecosystem, package_name
|
||||
|
||||
|
||||
def _manifest_path(report: dict[str, Any]) -> str:
|
||||
metadata = report.get("dependency_metadata")
|
||||
if not isinstance(metadata, dict):
|
||||
return ""
|
||||
return str(metadata.get("manifest_path") or "").strip()
|
||||
|
||||
|
||||
def _distinct_manifest_paths(candidate: dict[str, Any], report: dict[str, Any]) -> bool:
|
||||
"""Same CVE/package observed in two different manifests is two findings.
|
||||
|
||||
Only applies when both sides carry a manifest_path; a missing path keeps
|
||||
the legacy CVE/package/ecosystem identity.
|
||||
"""
|
||||
candidate_path = _manifest_path(candidate)
|
||||
report_path = _manifest_path(report)
|
||||
return bool(candidate_path and report_path and candidate_path != report_path)
|
||||
|
||||
|
||||
def _report_cve(report: dict[str, Any]) -> str:
|
||||
return str(report.get("cve") or "").strip().upper()
|
||||
|
||||
@@ -228,6 +246,8 @@ def _check_dependency_duplicate(
|
||||
report_cve, report_ecosystem, report_package_name = report_identity
|
||||
if (report_cve, report_package_name) != (cve, package_name):
|
||||
continue
|
||||
if _distinct_manifest_paths(candidate, report):
|
||||
continue
|
||||
if report_ecosystem == ecosystem:
|
||||
return {
|
||||
"is_duplicate": True,
|
||||
|
||||
@@ -531,6 +531,10 @@ def _result_properties(
|
||||
if value not in (None, ""):
|
||||
strix[key] = value
|
||||
|
||||
dependency_metadata = report.get("dependency_metadata")
|
||||
if isinstance(dependency_metadata, dict) and dependency_metadata:
|
||||
strix["dependency_metadata"] = dependency_metadata
|
||||
|
||||
# SARIF is written for external upload (code-scanning / ASPM), so it must
|
||||
# NOT carry the weaponized exploit payload — that stays a local run
|
||||
# artifact (vulnerabilities.json / the finding MD). We surface the PoC
|
||||
|
||||
@@ -205,6 +205,8 @@ def render_vulnerability_md(report: dict[str, Any]) -> str: # noqa: PLR0912, PL
|
||||
("Ecosystem", dep_meta.get("package_ecosystem")),
|
||||
("Installed Version", dep_meta.get("installed_version")),
|
||||
("Fixed Version", dep_meta.get("fixed_version")),
|
||||
("Introduced By", dep_meta.get("introduced_by")),
|
||||
("Dependency Chain", dep_meta.get("dependency_path")),
|
||||
("Endpoint", report.get("endpoint")),
|
||||
("Method", report.get("method")),
|
||||
("CVE", report.get("cve")),
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
---
|
||||
name: api_spec_testing
|
||||
description: Spec-driven API pentesting — systematically exercise every endpoint from an ingested OpenAPI/Swagger/Postman inventory for authz, injection, and business-logic flaws
|
||||
---
|
||||
|
||||
# API Spec Testing
|
||||
|
||||
When a target is an API specification (OpenAPI 3.x, Swagger 2.0, or a Postman
|
||||
collection), the root task lists it under **API Specifications** with the path
|
||||
to the spec file in the workspace and the authorized base URL(s). Read the spec
|
||||
file first and build your own endpoint inventory from it — every operation with
|
||||
its method, path, parameters, request-body schema (resolve `$ref`/`allOf`), and
|
||||
auth scheme. Do not rediscover the surface by crawling. Walk the inventory
|
||||
operation-by-operation and prove findings against the live base URL(s), which
|
||||
are authorized in scope.
|
||||
|
||||
## Methodology
|
||||
|
||||
**1. Baseline the contract.** For each endpoint, send a well-formed request that
|
||||
matches the declared schema and record the normal response (status, shape,
|
||||
auth requirement). This baseline is what every abuse case is compared against.
|
||||
|
||||
**2. Enumerate coverage.** Track every `METHOD path` in the inventory and mark it
|
||||
tested. Undocumented-but-implied siblings are worth probing too (e.g. if
|
||||
`GET /users/{id}` exists, try `PUT`/`DELETE`/`PATCH` on the same path even when
|
||||
the spec omits them — specs routinely under-document write operations).
|
||||
|
||||
**3. Prioritize by risk.** Object-scoped reads/writes, exports, admin/staff
|
||||
operations, and anything touching billing, auth, or PII first.
|
||||
|
||||
## What to test per endpoint
|
||||
|
||||
Test the full range of API weaknesses against each operation, driven by what the
|
||||
contract reveals — do not treat the following as an exhaustive checklist. The
|
||||
highest-yield classes on APIs are **authorization** flaws, since the spec hands
|
||||
you the object identifiers and privilege boundaries to abuse: examples include
|
||||
BOLA/IDOR (swap `{id}`/`accountId`/`tenantId` across two accounts), BFLA
|
||||
(privileged operations with a lower-privilege token), and missing/broken auth
|
||||
(replay with the token stripped or expired against endpoints whose declared auth
|
||||
says one is required). Beyond authorization, use the declared parameters and
|
||||
body schema as a launch point for mass assignment and excessive data exposure,
|
||||
injection and type-confusion on every parameter, and multi-step business-logic
|
||||
and rate-limit abuse — and follow the contract wherever it suggests something
|
||||
else worth probing.
|
||||
|
||||
## Validation
|
||||
|
||||
A finding is only real once reproduced against the live base URL with a
|
||||
concrete request/response pair. Capture the exact HTTP request (method, path,
|
||||
headers, body) and the response proving impact (another account's data, a
|
||||
privileged action succeeding, an injected payload executing). Prefer two-account
|
||||
diffs for authorization findings: same request, different token, unauthorized
|
||||
success.
|
||||
|
||||
## Tips
|
||||
|
||||
- The base URL(s) from the spec are authorized targets — send real traffic.
|
||||
- Path templates use `{param}`; substitute real values from your baseline.
|
||||
- For Postman collections, saved example values and environment variables are
|
||||
strong hints for valid inputs — use them to get past validation quickly.
|
||||
- Keep a running coverage table so no operation in the inventory is skipped.
|
||||
@@ -28,7 +28,7 @@ Run from the repo root and store output in the shared artifact directory used by
|
||||
the source-aware pass:
|
||||
|
||||
```bash
|
||||
ART=/workspace/.strix-source-aware
|
||||
ART=/workspace/.source-aware
|
||||
mkdir -p "$ART"
|
||||
|
||||
# Record the vuln DB age so a stale DB is a visible signal, not a silent clean scan.
|
||||
@@ -39,9 +39,11 @@ trivy version --format json 2>/dev/null | tee "$ART/trivy-version.json"
|
||||
# sandbox with egress gets the freshest CVEs; if the update fails, fall back to the
|
||||
# cached DB instead of failing the scan. --offline-scan keeps per-package advisory
|
||||
# lookups offline.
|
||||
trivy fs --scanners vuln --timeout 30m --offline-scan \
|
||||
# --list-all-pkgs includes the package graph (Relationship + DependsOn) needed
|
||||
# to attribute transitive CVEs to the direct dependency that introduces them.
|
||||
trivy fs --scanners vuln --timeout 30m --offline-scan --list-all-pkgs \
|
||||
--format json --output "$ART/trivy-sca.json" . \
|
||||
|| trivy fs --scanners vuln --timeout 30m --offline-scan --skip-db-update \
|
||||
|| trivy fs --scanners vuln --timeout 30m --offline-scan --skip-db-update --list-all-pkgs \
|
||||
--format json --output "$ART/trivy-sca.json" . \
|
||||
|| true
|
||||
```
|
||||
@@ -75,18 +77,111 @@ For each entry under `.Results[].Vulnerabilities[]` in `trivy-sca.json`, collect
|
||||
- `CVSS` — the published advisory base score
|
||||
- `PrimaryURL` / references — to verify the advisory
|
||||
|
||||
Deduplicate by `(CVE, PkgName, InstalledVersion)`. File one
|
||||
`create_dependency_report` per CVE — do not batch multiple CVEs into one report.
|
||||
Deduplicate by `(CVE, PkgName, Target)` — the same CVE/package observed in two
|
||||
different manifests (e.g. two workspaces of a monorepo) is two findings, one
|
||||
per manifest. File one `create_dependency_report` per CVE — do not batch
|
||||
multiple CVEs into one report.
|
||||
|
||||
### Attribute transitive CVEs to the direct dependency
|
||||
|
||||
With `--list-all-pkgs`, each `.Results[].Packages[]` entry carries `ID`
|
||||
(`name@version`), `Relationship` (`direct` / `indirect`) and `DependsOn` (the
|
||||
`ID`s it resolves to). For every vulnerable package that is **indirect**, walk
|
||||
the `DependsOn` graph backwards to find the `direct` package(s) whose closure
|
||||
contains it, then pass to `create_dependency_report`:
|
||||
|
||||
- `introduced_by` — the direct dependency as `name@version` (e.g.
|
||||
`express@4.18.1`). If several direct dependencies pull it in, pick the
|
||||
primary one and name the rest in `technical_analysis`.
|
||||
- `dependency_path` — the shortest resolution chain from that direct
|
||||
dependency to the vulnerable package, joined with ` > ` (e.g.
|
||||
`express@4.18.1 > body-parser@1.20.0 > qs@6.10.2`).
|
||||
- Omit both when the vulnerable package is itself a direct dependency.
|
||||
|
||||
If the ecosystem's lockfile gives trivy no graph (`DependsOn` absent), derive
|
||||
the chain from the package manager instead (`npm ls <pkg>`, `pnpm why <pkg>`,
|
||||
`yarn why <pkg>`, `pipdeptree --reverse -p <pkg>`, `go mod graph`,
|
||||
`mvn dependency:tree`, ...) — and if that also fails, leave the fields out
|
||||
rather than guessing.
|
||||
|
||||
For transitive findings, `remediation_steps` must be actionable at the
|
||||
**direct-dependency level**: upgrading the vulnerable package directly is
|
||||
usually impossible from the app's own manifest. Say which direct dependency to
|
||||
bump (a version whose closure resolves the fixed version), or how to force the
|
||||
resolution (npm `overrides` / yarn `resolutions` / pnpm `pnpm.overrides` /
|
||||
Maven `dependencyManagement` / Gradle resolution strategy / `go mod edit`),
|
||||
not just "upgrade <vulnerable pkg> to <fixed>".
|
||||
|
||||
### Usage / reachability analysis (required for every dependency CVE)
|
||||
|
||||
For every CVE you are about to report, run a static usage analysis and record
|
||||
the result in the structured `reachability` + `reachability_evidence` fields.
|
||||
The level is an **evidence ladder, never an exploitability verdict** — claim
|
||||
only what you proved, and cite the proof. It never changes severity (that is
|
||||
`advisory_cvss` alone); it exists so the reader can prioritize.
|
||||
|
||||
**Go — use govulncheck (real call-graph analysis):**
|
||||
|
||||
```bash
|
||||
# Symbol-level: reports only vulnerabilities whose vulnerable functions are
|
||||
# actually reachable from application code. Needs the Go toolchain + module
|
||||
# deps; if either is missing, fall back to the checks below rather than
|
||||
# claiming a level.
|
||||
if command -v govulncheck >/dev/null && go version >/dev/null 2>&1; then
|
||||
govulncheck -format json ./... > "$ART/govulncheck.json" || true
|
||||
fi
|
||||
```
|
||||
|
||||
- A finding with a call stack ⇒ `reachability=reachable_call_path`, put the
|
||||
call-path excerpt (entrypoint → vulnerable function) in
|
||||
`reachability_evidence`.
|
||||
- Listed as affecting a required module but with no reachable symbol ⇒ fall
|
||||
back to the import/symbol checks below (`imported` / `not_imported`).
|
||||
|
||||
**All other ecosystems — import check, then symbol match:**
|
||||
|
||||
1. **Import check.** Search application code (exclude lockfiles, vendored
|
||||
deps, `node_modules`, build output) for imports of the vulnerable package:
|
||||
`ast-grep`/`rg` for `import`/`require`/`from X import` of the package (and
|
||||
its ecosystem import name, which may differ from the registry name, e.g.
|
||||
`PyYAML` → `yaml`). No hits ⇒ `not_imported`, with the search scope stated
|
||||
in `reachability_evidence`. For a **transitive** dependency, the check is
|
||||
whether application code imports it directly; if not, it is reachable only
|
||||
through the direct dependency — check whether the direct dep's usage can
|
||||
hit it (if unclear, use `imported` when the direct dep is used at all).
|
||||
2. **Symbol match.** Read the advisory (GHSA/NVD/OSV `affected[].ecosystem_specific.imports` or the
|
||||
advisory text) for the affected functions/classes/APIs. Search application
|
||||
code for those symbols (`ast-grep` pattern or `rg -n`). Hits ⇒
|
||||
`vulnerable_symbol_used`, with repo-relative `file:line` of each hit (up
|
||||
to a handful) in `reachability_evidence`. Imported but no affected-symbol
|
||||
usage found (or the advisory names no symbols) ⇒ `imported`.
|
||||
3. If the analysis was not performed or is inconclusive (obfuscated code,
|
||||
dynamic loading, unparsable sources) ⇒ `unknown` and say why in
|
||||
`assumptions`.
|
||||
|
||||
Cheap-first budgeting: the import check is one search per package — always do
|
||||
it. Do the symbol match at least for every `critical`/`high`/KEV CVE; batch
|
||||
the searches. Never let this analysis stall reporting — `unknown` with a
|
||||
reason beats an unverified claim.
|
||||
|
||||
Anti-overclaim rules:
|
||||
|
||||
- `not_imported` still does NOT mean safe (dynamic `import()`/reflection/
|
||||
framework wiring evade static search) — never phrase it as "not exploitable".
|
||||
- `reachable_call_path` is reserved for call-graph tools (govulncheck); a
|
||||
symbol grep hit is `vulnerable_symbol_used`, no matter how convinced you are.
|
||||
- The tool rejects any level other than `unknown` without
|
||||
`reachability_evidence`.
|
||||
|
||||
### Reachability is a confidence modifier, not a gate
|
||||
|
||||
Do NOT suppress or downgrade a known CVE just because you could not prove the
|
||||
vulnerable code path is reachable. Report it, set `advisory_cvss` from the
|
||||
advisory, and use `assumptions` to note reachability (e.g. "the vulnerable
|
||||
`template()` API does not appear to be imported in application code, so practical
|
||||
exploitability is uncertain"). If you *can* show reachability or chain it into a
|
||||
dynamic exploit, do that and report it as a normal dynamic finding with
|
||||
`create_vulnerability_report` instead.
|
||||
advisory, record the usage analysis in `reachability`/`reachability_evidence`,
|
||||
and use `assumptions` for anything softer. If you *can* actually trigger the
|
||||
vulnerable path or chain it into a dynamic exploit, additionally report that
|
||||
as a normal dynamic finding with `create_vulnerability_report` (the standalone
|
||||
CVE stays in its own `create_dependency_report`).
|
||||
|
||||
## Reporting
|
||||
|
||||
@@ -107,6 +202,12 @@ findings and rejects empty PoC fields):
|
||||
- `package_ecosystem` — normalized ecosystem from `.Results[].Type` (lowercased,
|
||||
e.g. `npm`, `pypi`, `go`, `maven`, `rubygems`, `cargo`) (required).
|
||||
- `fixed_version` — `FixedVersion` (leave empty only if no fix is published).
|
||||
- `manifest_path` — the repo-relative `Target` lockfile/manifest path
|
||||
(required). Strip any scan-workspace or repo checkout directory prefix so
|
||||
the path is relative to the repository root (e.g. `package-lock.json`,
|
||||
`services/api/pom.xml`); the tool rejects absolute paths and `..` segments.
|
||||
This binds the finding to the exact file so remediation can target the
|
||||
right repository.
|
||||
- Reference the repo-relative `Target` lockfile path in `description` /
|
||||
`technical_analysis` (no leading slash) so the finding is traceable.
|
||||
- Put the concrete proof in `description` / `technical_analysis`: package name,
|
||||
@@ -120,7 +221,8 @@ findings and rejects empty PoC fields):
|
||||
- Set `cwe` to the most specific `CWE-NNN` when the advisory names one.
|
||||
- Do NOT cap severity at LOW just because there is no dynamic reproduction — use
|
||||
the advisory score.
|
||||
- Use `assumptions` for reachability/exploitability caveats.
|
||||
- Set `reachability` + `reachability_evidence` from the usage analysis above;
|
||||
use `assumptions` for anything softer (confidence, caveats, analysis limits).
|
||||
|
||||
Verify the CVE with `web_search` when available before reporting. Never guess or
|
||||
hallucinate a CVE id.
|
||||
@@ -136,3 +238,5 @@ hallucinate a CVE id.
|
||||
- Do not silently drop a known CVE because it lacks a dynamic PoC — that is the
|
||||
exact failure this skill prevents.
|
||||
- Do not downgrade advisory severity for lack of dynamic reproduction.
|
||||
- Do not claim a `reachability` level the evidence does not prove — `unknown`
|
||||
with a reason is always acceptable; an overclaimed level never is.
|
||||
|
||||
@@ -12,7 +12,7 @@ Use this skill for source-heavy analysis where static and structural signals sho
|
||||
Run tools from repo root and store outputs in a dedicated artifact directory:
|
||||
|
||||
```bash
|
||||
mkdir -p /workspace/.strix-source-aware
|
||||
mkdir -p /workspace/.source-aware
|
||||
```
|
||||
|
||||
## Baseline Coverage Bundle (Recommended)
|
||||
@@ -20,7 +20,7 @@ mkdir -p /workspace/.strix-source-aware
|
||||
Run this baseline once per repository before deep narrowing:
|
||||
|
||||
```bash
|
||||
ART=/workspace/.strix-source-aware
|
||||
ART=/workspace/.source-aware
|
||||
mkdir -p "$ART"
|
||||
|
||||
semgrep scan --config p/default --config p/golang --config p/secrets \
|
||||
@@ -30,7 +30,7 @@ python3 - <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
art = Path("/workspace/.strix-source-aware")
|
||||
art = Path("/workspace/.source-aware")
|
||||
semgrep_json = art / "semgrep.json"
|
||||
targets_file = art / "sg-targets.txt"
|
||||
|
||||
@@ -70,10 +70,10 @@ Use Semgrep as the default static triage pass:
|
||||
```bash
|
||||
# Preferred deterministic profile set (works with --metrics=off)
|
||||
semgrep scan --config p/default --config p/golang --config p/secrets \
|
||||
--metrics=off --json --output /workspace/.strix-source-aware/semgrep.json .
|
||||
--metrics=off --json --output /workspace/.source-aware/semgrep.json .
|
||||
|
||||
# If you choose auto config, do not combine it with --metrics=off
|
||||
semgrep scan --config auto --json --output /workspace/.strix-source-aware/semgrep-auto.json .
|
||||
semgrep scan --config auto --json --output /workspace/.source-aware/semgrep-auto.json .
|
||||
```
|
||||
|
||||
If diff scope is active, restrict to changed files first, then expand only when needed.
|
||||
@@ -85,8 +85,8 @@ Use `sg` for structure-aware code hunting:
|
||||
```bash
|
||||
# Ruleless structural pass over deterministic target list (no sgconfig.yml required)
|
||||
xargs -r -n 200 sg run --pattern '$F($$$ARGS)' --json=stream \
|
||||
< /workspace/.strix-source-aware/sg-targets.txt \
|
||||
> /workspace/.strix-source-aware/ast-grep.json 2> /workspace/.strix-source-aware/ast-grep.log || true
|
||||
< /workspace/.source-aware/sg-targets.txt \
|
||||
> /workspace/.source-aware/ast-grep.json 2> /workspace/.source-aware/ast-grep.log || true
|
||||
```
|
||||
|
||||
Target high-value patterns such as:
|
||||
@@ -110,15 +110,15 @@ Use outputs to improve route/symbol/sink maps for subsequent targeted scans.
|
||||
Detect hardcoded credentials:
|
||||
|
||||
```bash
|
||||
gitleaks detect --source . --report-format json --report-path /workspace/.strix-source-aware/gitleaks.json
|
||||
trufflehog filesystem --json . > /workspace/.strix-source-aware/trufflehog.json
|
||||
gitleaks detect --source . --report-format json --report-path /workspace/.source-aware/gitleaks.json
|
||||
trufflehog filesystem --json . > /workspace/.source-aware/trufflehog.json
|
||||
```
|
||||
|
||||
Run repository-wide dependency and config checks:
|
||||
|
||||
```bash
|
||||
trivy fs --scanners vuln,misconfig --timeout 30m --offline-scan \
|
||||
--format json --output /workspace/.strix-source-aware/trivy-fs.json . || true
|
||||
--format json --output /workspace/.source-aware/trivy-fs.json . || true
|
||||
```
|
||||
|
||||
Known-CVE dependency findings are the one exception to the "report only after
|
||||
@@ -132,9 +132,9 @@ For frontends and Node services, layer these on top of the language-agnostic
|
||||
passes above:
|
||||
|
||||
```bash
|
||||
retire --path . --outputformat json --outputpath /workspace/.strix-source-aware/retire.json || true
|
||||
retire --path . --outputformat json --outputpath /workspace/.source-aware/retire.json || true
|
||||
eslint --no-config-lookup --rule '{"no-eval":2,"no-implied-eval":2}' \
|
||||
-f json -o /workspace/.strix-source-aware/eslint.json . || true
|
||||
-f json -o /workspace/.source-aware/eslint.json . || true
|
||||
```
|
||||
|
||||
When you hit a minified bundle, run `js-beautify <file>` for a readable
|
||||
|
||||
@@ -202,7 +202,7 @@ Confirm with a version/patch check before firing — these are destructive.
|
||||
|
||||
## Tooling
|
||||
|
||||
**None of the AD tools below ship in the Strix sandbox by default** (the image is Kali-rolling but installs only web-focused tooling). Install what the task needs — the sandbox has `pipx`, `pip`, `go`, `git`, and Kali's apt repos. AD testing also requires **network reachability to the target DC/subnet**, which the default web-target sandbox usually lacks; confirm connectivity first.
|
||||
**None of the AD tools below ship in the sandbox by default** (the image is Kali-rolling but installs only web-focused tooling). Install what the task needs — the sandbox has `pipx`, `pip`, `go`, `git`, and Kali's apt repos. AD testing also requires **network reachability to the target DC/subnet**, which the default web-target sandbox usually lacks; confirm connectivity first.
|
||||
|
||||
```
|
||||
# Python identity toolkit (impacket = GetUserSPNs/GetNPUsers/secretsdump/ntlmrelayx/getST/addcomputer/rbcd)
|
||||
|
||||
@@ -5,7 +5,7 @@ description: Run Python through exec_command in the SDK sandbox. Use the image-b
|
||||
|
||||
# Python In The Sandbox
|
||||
|
||||
Use `exec_command` for Python. There is no separate Strix Python executor.
|
||||
Use `exec_command` for Python. There is no separate Python executor.
|
||||
|
||||
Prefer writing reusable scripts to a `.py` file and running them with
|
||||
`python3 <name>.py`. For short one-off transformations, `python3 -c` or a
|
||||
|
||||
@@ -80,7 +80,7 @@ Gadget availability depends on package versions — enumerate `node_modules` in
|
||||
1. **Identify merge points** — Search for extend/merge/defaults/deep copy on user-controlled objects
|
||||
2. **Baseline probe** — Inject benign pollution marker:
|
||||
```json
|
||||
{"__proto__": {"strixPolluted": "yes"}}
|
||||
{"__proto__": {"pollutionCanary": "yes"}}
|
||||
```
|
||||
Verify via response behavior, error messages, or follow-up request reading shared state
|
||||
3. **Shape variants** — Test `__proto__`, `constructor.prototype`, nested bracket notation
|
||||
@@ -121,7 +121,7 @@ Gadget availability depends on package versions — enumerate `node_modules` in
|
||||
|
||||
## Pro Tips
|
||||
|
||||
1. Always verify pollution with a unique canary key (`strixPolluted_<random>`) before attempting RCE gadgets
|
||||
1. Always verify pollution with a unique canary key (`pollutionCanary_<random>`) before attempting RCE gadgets
|
||||
2. In white-box scans, grep for `merge`, `extend`, `defaultsDeep`, `assign` with user input
|
||||
3. Check both request parsing and response template config merges (second-order)
|
||||
4. Node gadget chains are version-specific — confirm package version before claiming RCE
|
||||
|
||||
@@ -14,6 +14,7 @@ from agents import RunContextWrapper, function_tool
|
||||
|
||||
from strix.core.agents import Status, coordinator_from_context
|
||||
from strix.core.execution import notify_parent_on_terminal
|
||||
from strix.core.hooks import LLM_TURN_KEY
|
||||
from strix.skills import validate_requested_skills
|
||||
|
||||
|
||||
@@ -224,6 +225,7 @@ _WAIT_DEFAULT_TIMEOUT_S = 300
|
||||
# ``timeout_seconds`` the model asks for. One second of headroom lets the
|
||||
# tool's own timeout fire first and return a clean result.
|
||||
_WAIT_HARD_CEILING_S = _WAIT_DEFAULT_TIMEOUT_S + 1
|
||||
_WAITED_TURN_KEY = "waited_llm_turn"
|
||||
|
||||
|
||||
@function_tool(timeout=_WAIT_HARD_CEILING_S)
|
||||
@@ -239,6 +241,11 @@ async def wait_for_agents( # noqa: PLR0911
|
||||
completion reports. You resume the instant any message arrives, so
|
||||
size ``timeout_seconds`` to the work you're awaiting.
|
||||
|
||||
**Issue exactly one wait, then stop and react to what it returns.**
|
||||
This call blocks and resumes on its own; it is not a poll you repeat.
|
||||
Do not write out a wait/check loop ahead of time — a second wait in
|
||||
the same turn returns immediately without waiting.
|
||||
|
||||
**This tool is only for waiting on other agents.** Two things it is
|
||||
NOT for:
|
||||
|
||||
@@ -290,6 +297,24 @@ async def wait_for_agents( # noqa: PLR0911
|
||||
default=str,
|
||||
)
|
||||
|
||||
turn = inner.get(LLM_TURN_KEY)
|
||||
if turn is not None and inner.get(_WAITED_TURN_KEY) == turn:
|
||||
return json.dumps(
|
||||
{
|
||||
"success": True,
|
||||
"wait_outcome": "already_waited",
|
||||
"reason": reason,
|
||||
"note": (
|
||||
"You already waited in this turn. A single wait_for_agents blocks and "
|
||||
"resumes on its own, so queueing more waits only strands you — issue one "
|
||||
"wait, then react to what it returns."
|
||||
),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
default=str,
|
||||
)
|
||||
inner[_WAITED_TURN_KEY] = turn
|
||||
|
||||
async with coordinator._lock:
|
||||
stopped = coordinator.statuses.get(me) == "stopped"
|
||||
if stopped:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Bound oversized tool results before they enter agent history.
|
||||
|
||||
Oversized results are spilled into the sandbox at
|
||||
``/workspace/.strix/tool-output/<id>.txt``; the agent sees a head + tail slice
|
||||
``/workspace/.tool-output/<id>.txt``; the agent sees a head + tail slice
|
||||
plus the path and reads the rest back with its own file tools. The spill writer
|
||||
is injected by the runner via :func:`configure_spill_writer`.
|
||||
"""
|
||||
@@ -25,7 +25,7 @@ _WORKSPACE_SPILL_NOTICE = (
|
||||
"in the sandbox; read it with exec_command (e.g. `sed -n`, `grep`, `cat`) ...]"
|
||||
)
|
||||
|
||||
WORKSPACE_SPILL_DIR = "/workspace/.strix/tool-output"
|
||||
WORKSPACE_SPILL_DIR = "/workspace/.tool-output"
|
||||
|
||||
# Longest possible workspace path, used only to reserve notice bytes.
|
||||
_SAMPLE_WORKSPACE_PATH = f"{WORKSPACE_SPILL_DIR}/{'0' * 32}.txt"
|
||||
|
||||
@@ -189,7 +189,11 @@ def build_raw_request(
|
||||
|
||||
final_headers = {**headers}
|
||||
final_headers.setdefault("Host", parsed.netloc)
|
||||
final_headers.setdefault("User-Agent", "strix")
|
||||
final_headers.setdefault(
|
||||
"User-Agent",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
)
|
||||
# Framing headers inherited from the captured request describe the ORIGINAL
|
||||
# body; once the body is modified for replay they are stale. We always send a
|
||||
# plain (non-chunked) body with an explicit Content-Length, so drop any
|
||||
|
||||
@@ -719,12 +719,47 @@ def _dependency_severity(advisory_cvss: float | None) -> tuple[float, str]:
|
||||
return score, "none"
|
||||
|
||||
|
||||
_VALID_REACHABILITY = frozenset(
|
||||
{
|
||||
"not_imported",
|
||||
"imported",
|
||||
"vulnerable_symbol_used",
|
||||
"reachable_call_path",
|
||||
"unknown",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _validate_manifest_path(manifest_path: str | None) -> str | None:
|
||||
"""Return an error message when manifest_path is missing or unsafe."""
|
||||
path = (manifest_path or "").strip()
|
||||
if not path:
|
||||
return (
|
||||
"manifest_path is required: pass the repo-relative path of the "
|
||||
"lockfile/manifest where the vulnerable version was observed "
|
||||
"(trivy's Target, e.g. 'package-lock.json' or "
|
||||
"'services/api/pom.xml'). It binds the finding to its exact file "
|
||||
"so remediation can target the right repository."
|
||||
)
|
||||
if path.startswith("/") or "\\" in path or path.split("/")[0].endswith(":"):
|
||||
return f"manifest_path must be a relative path within the repository, got {path!r}"
|
||||
segments = path.split("/")
|
||||
if any(segment in ("", ".", "..") for segment in segments):
|
||||
return f"manifest_path must not contain empty, '.', or '..' segments, got {path!r}"
|
||||
return None
|
||||
|
||||
|
||||
def _build_dependency_metadata(
|
||||
*,
|
||||
package_name: str,
|
||||
installed_version: str,
|
||||
package_ecosystem: str | None,
|
||||
fixed_version: str | None,
|
||||
introduced_by: str | None,
|
||||
dependency_path: str | None,
|
||||
manifest_path: str | None = None,
|
||||
reachability: str | None = None,
|
||||
reachability_evidence: str | None = None,
|
||||
) -> dict[str, str]:
|
||||
metadata = {
|
||||
"package_name": package_name.strip(),
|
||||
@@ -732,17 +767,43 @@ def _build_dependency_metadata(
|
||||
}
|
||||
if package_ecosystem and package_ecosystem.strip():
|
||||
metadata["package_ecosystem"] = package_ecosystem.strip()
|
||||
if manifest_path and manifest_path.strip():
|
||||
metadata["manifest_path"] = manifest_path.strip()
|
||||
if fixed_version and fixed_version.strip():
|
||||
metadata["fixed_version"] = fixed_version.strip()
|
||||
if introduced_by and introduced_by.strip():
|
||||
metadata["introduced_by"] = introduced_by.strip()
|
||||
if dependency_path and dependency_path.strip():
|
||||
metadata["dependency_path"] = dependency_path.strip()
|
||||
# "unknown" is the absent case — omitting it keeps the jsonb contract clean,
|
||||
# and evidence without a level would have nothing to qualify.
|
||||
if reachability and reachability.strip() and reachability.strip() != "unknown":
|
||||
metadata["reachability"] = reachability.strip()
|
||||
if reachability_evidence and reachability_evidence.strip():
|
||||
metadata["reachability_evidence"] = reachability_evidence.strip()
|
||||
return metadata
|
||||
|
||||
|
||||
_REACHABILITY_EVIDENCE_LABELS = {
|
||||
"not_imported": "not imported by application code",
|
||||
"imported": "imported by application code; affected API usage unconfirmed",
|
||||
"vulnerable_symbol_used": "the advisory's affected API is used in application code",
|
||||
"reachable_call_path": (
|
||||
"a call path from application code to the vulnerable function was proven"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _build_dependency_evidence(
|
||||
*,
|
||||
cve: str,
|
||||
package_name: str,
|
||||
installed_version: str,
|
||||
fixed_version: str | None,
|
||||
introduced_by: str | None,
|
||||
dependency_path: str | None,
|
||||
reachability: str | None = None,
|
||||
reachability_evidence: str | None = None,
|
||||
) -> str:
|
||||
evidence = (
|
||||
f"**Advisory evidence:** `{cve}` applies to `{package_name}` "
|
||||
@@ -750,6 +811,22 @@ def _build_dependency_evidence(
|
||||
)
|
||||
if fixed_version and fixed_version.strip():
|
||||
evidence += f" The advisory is fixed in `{fixed_version.strip()}`."
|
||||
if introduced_by and introduced_by.strip():
|
||||
evidence += (
|
||||
f"\n\n**Transitive dependency:** introduced by the direct "
|
||||
f"dependency `{introduced_by.strip()}`."
|
||||
)
|
||||
if dependency_path and dependency_path.strip():
|
||||
evidence += f"\n\n**Dependency chain:** `{dependency_path.strip()}`"
|
||||
label = _REACHABILITY_EVIDENCE_LABELS.get((reachability or "").strip().lower())
|
||||
if label:
|
||||
evidence += f"\n\n**Usage analysis:** {label}."
|
||||
if reachability_evidence and reachability_evidence.strip():
|
||||
evidence += f" {reachability_evidence.strip()}"
|
||||
evidence += (
|
||||
" This is a prioritization signal from static analysis, not a"
|
||||
" proof of exploitability or of safety."
|
||||
)
|
||||
return evidence
|
||||
|
||||
|
||||
@@ -770,6 +847,11 @@ async def _do_create_dependency( # noqa: PLR0912
|
||||
advisory_cvss: float | None,
|
||||
technical_analysis: str | None,
|
||||
fix_effort: str,
|
||||
introduced_by: str | None = None,
|
||||
dependency_path: str | None = None,
|
||||
manifest_path: str | None = None,
|
||||
reachability: str = "unknown",
|
||||
reachability_evidence: str | None = None,
|
||||
agent_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
@@ -806,6 +888,22 @@ async def _do_create_dependency( # noqa: PLR0912
|
||||
f"Invalid fix_effort: {fix_effort!r}. Must be one of: {sorted(_VALID_FIX_EFFORT)}"
|
||||
)
|
||||
|
||||
manifest_err = _validate_manifest_path(manifest_path)
|
||||
if manifest_err:
|
||||
errors.append(manifest_err)
|
||||
|
||||
reachability = (reachability or "unknown").strip().lower()
|
||||
if reachability not in _VALID_REACHABILITY:
|
||||
errors.append(
|
||||
f"Invalid reachability: {reachability!r}. Must be one of: {sorted(_VALID_REACHABILITY)}"
|
||||
)
|
||||
elif reachability != "unknown" and not (reachability_evidence or "").strip():
|
||||
errors.append(
|
||||
"reachability_evidence is required when reachability is not 'unknown': "
|
||||
"cite the concrete proof (import file:line, matched symbol usage, or "
|
||||
"govulncheck call path). Never claim a reachability level without evidence."
|
||||
)
|
||||
|
||||
if advisory_cvss is None:
|
||||
errors.append(
|
||||
"advisory_cvss is required: read the published advisory base score "
|
||||
@@ -824,12 +922,21 @@ async def _do_create_dependency( # noqa: PLR0912
|
||||
installed_version=installed_version,
|
||||
package_ecosystem=package_ecosystem,
|
||||
fixed_version=fixed_version,
|
||||
introduced_by=introduced_by,
|
||||
dependency_path=dependency_path,
|
||||
manifest_path=manifest_path,
|
||||
reachability=reachability,
|
||||
reachability_evidence=reachability_evidence,
|
||||
)
|
||||
evidence = _build_dependency_evidence(
|
||||
cve=parsed_cve,
|
||||
package_name=package_name.strip(),
|
||||
installed_version=installed_version.strip(),
|
||||
fixed_version=fixed_version,
|
||||
introduced_by=introduced_by,
|
||||
dependency_path=dependency_path,
|
||||
reachability=reachability,
|
||||
reachability_evidence=reachability_evidence,
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -922,10 +1029,15 @@ async def create_dependency_report(
|
||||
remediation_steps: str,
|
||||
assumptions: str,
|
||||
package_ecosystem: str,
|
||||
manifest_path: str | None = None,
|
||||
fixed_version: str | None = None,
|
||||
cwe: str | None = None,
|
||||
technical_analysis: str | None = None,
|
||||
fix_effort: str = "low",
|
||||
introduced_by: str | None = None,
|
||||
dependency_path: str | None = None,
|
||||
reachability: str = "unknown",
|
||||
reachability_evidence: str | None = None,
|
||||
) -> str:
|
||||
"""File a known-CVE dependency (SCA) finding — one report per CVE x package.
|
||||
|
||||
@@ -950,9 +1062,26 @@ async def create_dependency_report(
|
||||
- Re-reporting the same CVE/package already filed.
|
||||
|
||||
**Reachability**: do NOT silently downgrade or suppress a finding
|
||||
because the vulnerable code path may be unreachable — instead state
|
||||
reachability as an ``assumptions`` / confidence factor. Report the
|
||||
finding; let the reader weigh exploitability.
|
||||
because the vulnerable code path may be unreachable — report it, and
|
||||
record what the usage analysis showed via the structured
|
||||
``reachability`` + ``reachability_evidence`` fields (see the
|
||||
dependency-cve-scanning skill for the analysis procedure). The level
|
||||
is an evidence ladder, never an exploitability verdict:
|
||||
|
||||
- ``not_imported`` — the package is never imported/required by
|
||||
application code (strongest de-prioritization signal; still not
|
||||
proof of safety — dynamic loading, reflection, or framework wiring
|
||||
can evade static search).
|
||||
- ``imported`` — application code imports the package, but usage of
|
||||
the advisory's affected API was not confirmed.
|
||||
- ``vulnerable_symbol_used`` — the advisory's affected
|
||||
function/class/API appears in application code.
|
||||
- ``reachable_call_path`` — a call-graph tool (e.g. ``govulncheck``)
|
||||
proved a path from application code to the vulnerable function.
|
||||
- ``unknown`` — usage analysis was not performed or was inconclusive.
|
||||
|
||||
Severity is still derived solely from ``advisory_cvss`` — the
|
||||
reachability level never changes the rating, only prioritization.
|
||||
|
||||
**Formatting**: use markdown in text fields (``**bold**``, ``inline
|
||||
code`` for package/version identifiers, fenced code blocks for
|
||||
@@ -978,6 +1107,30 @@ async def create_dependency_report(
|
||||
technical_analysis: Optional deeper mechanism/root-cause detail.
|
||||
fix_effort: One of ``trivial`` / ``low`` / ``medium`` / ``high``
|
||||
(dependency upgrades are usually ``trivial``/``low``).
|
||||
introduced_by: For a **transitive** dependency, the direct
|
||||
dependency (from the project's own manifest) that pulls the
|
||||
vulnerable package in, as ``name@version`` (e.g.
|
||||
``express@4.18.1``). Omit when the vulnerable package is
|
||||
itself a direct dependency.
|
||||
dependency_path: The resolution chain from the direct dependency
|
||||
to the vulnerable package, joined with `` > `` (e.g.
|
||||
``express@4.18.1 > body-parser@1.20.0 > qs@6.10.2``). Omit
|
||||
for direct dependencies.
|
||||
manifest_path: **Required.** The repo-relative path of the
|
||||
lockfile/manifest where the vulnerable version was observed —
|
||||
trivy's ``Target`` (e.g. ``package-lock.json``,
|
||||
``services/api/pom.xml``). Strip any scan-workspace or repo
|
||||
checkout directory prefix so the path is relative to the
|
||||
repository root. This binds the finding to its exact file so
|
||||
remediation can target the right repository.
|
||||
reachability: Usage-evidence level from static analysis — one of
|
||||
``not_imported`` / ``imported`` / ``vulnerable_symbol_used`` /
|
||||
``reachable_call_path`` / ``unknown``. Claim only what the
|
||||
evidence proves; when in doubt use ``unknown``.
|
||||
reachability_evidence: The concrete proof for the claimed level
|
||||
(required for any level other than ``unknown``): repo-relative
|
||||
``file:line`` of the import or symbol usage, the matched
|
||||
advisory symbols, or the govulncheck call-path excerpt.
|
||||
"""
|
||||
agent_id, agent_name = _caller_identity(ctx)
|
||||
|
||||
@@ -997,6 +1150,11 @@ async def create_dependency_report(
|
||||
advisory_cvss=advisory_cvss,
|
||||
technical_analysis=technical_analysis,
|
||||
fix_effort=fix_effort,
|
||||
introduced_by=introduced_by,
|
||||
dependency_path=dependency_path,
|
||||
manifest_path=manifest_path,
|
||||
reachability=reachability,
|
||||
reachability_evidence=reachability_evidence,
|
||||
agent_id=agent_id,
|
||||
agent_name=agent_name,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
"""Recognize API specifications and extract the hosts they declare.
|
||||
|
||||
Supports OpenAPI 3.x, Swagger 2.0, and Postman Collection v2.1. Two things about
|
||||
an API spec must be decided on the host, in code: whether a target file is a
|
||||
spec at all (detection), and which base URLs it authorizes as in-scope hosts
|
||||
(scope cannot be self-granted by the agent). Everything else about the contract
|
||||
— operations, parameters, request bodies, auth — is left to the agent, which
|
||||
reads the spec file directly in the sandbox, so ``$ref``, ``allOf``, and nested
|
||||
schemas resolve properly instead of being re-parsed here. Collections held only
|
||||
in Postman are fetched here too, so the API key stays on the host and never
|
||||
enters the sandbox.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
SPEC_EXTENSIONS = frozenset({".json", ".yaml", ".yml"})
|
||||
|
||||
#: Guard against pathological Postman folder nesting.
|
||||
_MAX_POSTMAN_DEPTH = 25
|
||||
|
||||
|
||||
class SpecParseError(ValueError):
|
||||
"""Raised when a spec cannot be read, recognized, or fetched."""
|
||||
|
||||
|
||||
def load_spec(path: str | Path) -> dict[str, Any]:
|
||||
"""Load an API spec file as a mapping.
|
||||
|
||||
Raises :class:`SpecParseError` if the file cannot be read or is not a
|
||||
JSON/YAML mapping.
|
||||
"""
|
||||
p = Path(path)
|
||||
try:
|
||||
text = p.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
raise SpecParseError(f"Cannot read spec {p}: {exc}") from exc
|
||||
# JSON is a subset of YAML, so safe_load parses both; try JSON first for a
|
||||
# clearer error and to keep the fast path fast.
|
||||
try:
|
||||
data: Any = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
try:
|
||||
data = yaml.safe_load(text)
|
||||
except yaml.YAMLError as exc:
|
||||
raise SpecParseError(f"{p} is not valid JSON or YAML: {exc}") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise SpecParseError(f"{p} does not contain a mapping at the top level")
|
||||
return data
|
||||
|
||||
|
||||
def classify_spec(raw: dict[str, Any]) -> str | None:
|
||||
"""Return ``openapi`` / ``swagger`` / ``postman``, or ``None`` if unrecognized."""
|
||||
if isinstance(raw.get("openapi"), str):
|
||||
return "openapi"
|
||||
if str(raw.get("swagger", "")).startswith("2"):
|
||||
return "swagger"
|
||||
info = raw.get("info")
|
||||
if isinstance(info, dict) and ("_postman_id" in info or "item" in raw):
|
||||
return "postman"
|
||||
return None
|
||||
|
||||
|
||||
def detect_spec_format(path: Path) -> str | None:
|
||||
"""Return the spec format of *path*, or ``None`` if it is not a spec.
|
||||
|
||||
Only files whose extension is in :data:`SPEC_EXTENSIONS` are inspected; the
|
||||
contents are then loaded to confirm, so an arbitrary ``.json`` config is not
|
||||
mistaken for a spec.
|
||||
"""
|
||||
if path.suffix.lower() not in SPEC_EXTENSIONS:
|
||||
return None
|
||||
try:
|
||||
raw = load_spec(path)
|
||||
except SpecParseError:
|
||||
return None
|
||||
return classify_spec(raw)
|
||||
|
||||
|
||||
def spec_title(raw: dict[str, Any]) -> str:
|
||||
"""Return the spec's declared name, for display in the task and run record."""
|
||||
info = raw.get("info")
|
||||
if not isinstance(info, dict):
|
||||
return "API"
|
||||
name = info.get("title") or info.get("name") or "API"
|
||||
return str(name).strip() or "API"
|
||||
|
||||
|
||||
def _absolute_urls(candidates: list[str]) -> list[str]:
|
||||
"""Keep absolute http(s) URLs, without trailing slashes, in declared order."""
|
||||
urls: list[str] = []
|
||||
for candidate in candidates:
|
||||
split = urlsplit(candidate.strip())
|
||||
if split.scheme in ("http", "https") and split.netloc:
|
||||
urls.append(candidate.strip().rstrip("/"))
|
||||
return list(dict.fromkeys(urls))
|
||||
|
||||
|
||||
_SERVER_VAR_PATTERN = re.compile(r"\{([^{}/]+)\}")
|
||||
|
||||
|
||||
def _resolve_server_url(url: str, variables: Any) -> str:
|
||||
"""Substitute an OpenAPI server template's variables with their defaults."""
|
||||
if "{" not in url or not isinstance(variables, dict):
|
||||
return url
|
||||
defaults: dict[str, str] = {}
|
||||
for name, spec in variables.items():
|
||||
if isinstance(spec, dict) and spec.get("default") is not None:
|
||||
defaults[str(name)] = str(spec["default"])
|
||||
return _SERVER_VAR_PATTERN.sub(lambda m: defaults.get(m.group(1), m.group(0)), url)
|
||||
|
||||
|
||||
def _openapi_base_urls(raw: dict[str, Any]) -> list[str]:
|
||||
servers = raw.get("servers")
|
||||
if not isinstance(servers, list):
|
||||
return []
|
||||
return _absolute_urls(
|
||||
[
|
||||
_resolve_server_url(str(server["url"]), server.get("variables"))
|
||||
for server in servers
|
||||
if isinstance(server, dict) and server.get("url")
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _swagger_base_urls(raw: dict[str, Any]) -> list[str]:
|
||||
host = str(raw.get("host", "")).strip()
|
||||
if not host:
|
||||
return []
|
||||
base_path = str(raw.get("basePath", "")).strip()
|
||||
schemes = [s for s in (raw.get("schemes") or ["https"]) if isinstance(s, str)]
|
||||
return _absolute_urls([f"{scheme}://{host}{base_path}" for scheme in schemes])
|
||||
|
||||
|
||||
_POSTMAN_VAR_PATTERN = re.compile(r"\{\{\s*([^}]+?)\s*\}\}")
|
||||
|
||||
|
||||
def postman_variables(raw: dict[str, Any]) -> dict[str, str]:
|
||||
"""Build a ``{name: value}`` map from a Postman ``variable`` block."""
|
||||
variables: dict[str, str] = {}
|
||||
entries = raw.get("variable")
|
||||
if isinstance(entries, list):
|
||||
for entry in entries:
|
||||
if isinstance(entry, dict) and entry.get("key") is not None:
|
||||
variables[str(entry["key"])] = str(entry.get("value", ""))
|
||||
return variables
|
||||
|
||||
|
||||
def _resolve_postman_vars(text: str, variables: dict[str, str]) -> str:
|
||||
if not variables or "{{" not in text:
|
||||
return text
|
||||
return _POSTMAN_VAR_PATTERN.sub(lambda m: variables.get(m.group(1), m.group(0)), text)
|
||||
|
||||
|
||||
def _postman_request_url(url: Any, variables: dict[str, str]) -> str:
|
||||
if isinstance(url, str):
|
||||
raw = url
|
||||
elif isinstance(url, dict):
|
||||
raw = str(url.get("raw", ""))
|
||||
if not raw:
|
||||
host = url.get("host")
|
||||
raw = ".".join(str(h) for h in host) if isinstance(host, list) else str(host or "")
|
||||
else:
|
||||
return ""
|
||||
return _resolve_postman_vars(raw, variables)
|
||||
|
||||
|
||||
def _walk_postman_hosts(
|
||||
items: Any,
|
||||
variables: dict[str, str],
|
||||
hosts: list[str],
|
||||
depth: int = 0,
|
||||
) -> None:
|
||||
if depth > _MAX_POSTMAN_DEPTH or not isinstance(items, list):
|
||||
return
|
||||
for node in items:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
if isinstance(node.get("item"), list):
|
||||
_walk_postman_hosts(node["item"], variables, hosts, depth + 1)
|
||||
continue
|
||||
request = node.get("request")
|
||||
if not isinstance(request, dict):
|
||||
continue
|
||||
url = _postman_request_url(request.get("url"), variables)
|
||||
split = urlsplit(url)
|
||||
if split.scheme and split.netloc:
|
||||
hosts.append(f"{split.scheme}://{split.netloc}")
|
||||
|
||||
|
||||
def _postman_base_urls(raw: dict[str, Any], extra_variables: dict[str, str] | None) -> list[str]:
|
||||
variables = postman_variables(raw)
|
||||
if extra_variables:
|
||||
variables.update(extra_variables) # environment values override collection defaults
|
||||
hosts: list[str] = []
|
||||
_walk_postman_hosts(raw.get("item"), variables, hosts)
|
||||
return _absolute_urls(sorted(set(hosts)))
|
||||
|
||||
|
||||
def spec_base_urls(
|
||||
raw: dict[str, Any],
|
||||
*,
|
||||
extra_variables: dict[str, str] | None = None,
|
||||
) -> list[str]:
|
||||
"""Return the absolute base URLs a spec declares, for scope authorization.
|
||||
|
||||
Relative and unresolved-template URLs are dropped: an unusable value would
|
||||
otherwise be authorized as an in-scope host. Callers pair the spec with an
|
||||
explicit ``--target`` host when the spec declares none.
|
||||
"""
|
||||
spec_format = classify_spec(raw)
|
||||
if spec_format == "openapi":
|
||||
return _openapi_base_urls(raw)
|
||||
if spec_format == "swagger":
|
||||
return _swagger_base_urls(raw)
|
||||
if spec_format == "postman":
|
||||
return _postman_base_urls(raw, extra_variables)
|
||||
raise SpecParseError("File is not a recognized OpenAPI, Swagger, or Postman spec")
|
||||
|
||||
|
||||
POSTMAN_API_BASE = "https://api.getpostman.com"
|
||||
_POSTMAN_FETCH_TIMEOUT = 30
|
||||
|
||||
|
||||
def _postman_api_json(url: str, api_key: str, label: str) -> dict[str, Any]:
|
||||
"""GET a Postman API resource and return the parsed JSON payload.
|
||||
|
||||
Raises :class:`SpecParseError` with an actionable message on auth, network,
|
||||
or shape errors.
|
||||
"""
|
||||
if not api_key:
|
||||
raise SpecParseError(
|
||||
"POSTMAN_API_KEY is not set. Export a Postman API key (PMAK-…) to "
|
||||
"fetch from the Postman API, or pass a local collection file instead.",
|
||||
)
|
||||
try:
|
||||
response = requests.get(
|
||||
url,
|
||||
headers={"X-Api-Key": api_key, "Accept": "application/json"},
|
||||
timeout=_POSTMAN_FETCH_TIMEOUT,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
raise SpecParseError(f"Failed to reach the Postman API: {exc}") from exc
|
||||
|
||||
if response.status_code == 401:
|
||||
raise SpecParseError("Postman API rejected the key (401). Check POSTMAN_API_KEY.")
|
||||
if response.status_code == 404:
|
||||
raise SpecParseError(
|
||||
f"Postman {label} not found (404). Check the id and that the key can access it.",
|
||||
)
|
||||
if response.status_code != 200:
|
||||
raise SpecParseError(f"Postman API returned HTTP {response.status_code} for {label}.")
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise SpecParseError(f"Postman API returned non-JSON for {label}") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise SpecParseError(f"Unexpected Postman API response shape for {label}")
|
||||
return payload
|
||||
|
||||
|
||||
def fetch_postman_collection(collection_uid: str, api_key: str) -> dict[str, Any]:
|
||||
"""Fetch a collection from the Postman API and return the raw collection dict.
|
||||
|
||||
Uses ``GET /collections/{uid}`` with the ``X-Api-Key`` header. The endpoint
|
||||
wraps the collection under a ``collection`` key, unwrapped here so the result
|
||||
matches an exported collection file.
|
||||
"""
|
||||
payload = _postman_api_json(
|
||||
f"{POSTMAN_API_BASE}/collections/{collection_uid}",
|
||||
api_key,
|
||||
f"collection {collection_uid}",
|
||||
)
|
||||
collection = payload.get("collection", payload)
|
||||
if not isinstance(collection, dict) or not collection:
|
||||
raise SpecParseError(f"Postman collection {collection_uid} came back empty")
|
||||
return collection
|
||||
|
||||
|
||||
def fetch_postman_environment(environment_uid: str, api_key: str) -> dict[str, str]:
|
||||
"""Fetch a Postman environment and return its enabled ``{key: value}`` pairs.
|
||||
|
||||
Disabled values are skipped, matching how Postman resolves an environment at
|
||||
request time.
|
||||
"""
|
||||
payload = _postman_api_json(
|
||||
f"{POSTMAN_API_BASE}/environments/{environment_uid}",
|
||||
api_key,
|
||||
f"environment {environment_uid}",
|
||||
)
|
||||
environment = payload.get("environment", payload)
|
||||
values = environment.get("values") if isinstance(environment, dict) else None
|
||||
if not isinstance(values, list):
|
||||
return {}
|
||||
return {
|
||||
str(value["key"]): str(value.get("value", ""))
|
||||
for value in values
|
||||
if isinstance(value, dict) and value.get("key") and value.get("enabled", True)
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
"""Tests for spec recognition and base-URL extraction in strix.utils.api_spec."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
from strix.utils.api_spec import (
|
||||
SpecParseError,
|
||||
classify_spec,
|
||||
detect_spec_format,
|
||||
fetch_postman_collection,
|
||||
fetch_postman_environment,
|
||||
load_spec,
|
||||
spec_base_urls,
|
||||
spec_title,
|
||||
)
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
OPENAPI_YAML = """
|
||||
openapi: 3.0.1
|
||||
info:
|
||||
title: Shop API
|
||||
version: 1.0.0
|
||||
servers:
|
||||
- url: https://{region}.api.shop.test/{ver}
|
||||
variables:
|
||||
region:
|
||||
default: eu
|
||||
ver:
|
||||
default: v1
|
||||
paths:
|
||||
/users/{id}:
|
||||
get:
|
||||
summary: Get user
|
||||
"""
|
||||
|
||||
SWAGGER_JSON = {
|
||||
"swagger": "2.0",
|
||||
"info": {"title": "Legacy"},
|
||||
"host": "legacy.test",
|
||||
"basePath": "/api",
|
||||
"schemes": ["https"],
|
||||
"paths": {"/orders": {"post": {"summary": "Create order"}}},
|
||||
}
|
||||
|
||||
POSTMAN_JSON = {
|
||||
"info": {"_postman_id": "abc-123", "name": "Pet Store"},
|
||||
"item": [
|
||||
{
|
||||
"name": "Pets",
|
||||
"item": [
|
||||
{
|
||||
"name": "List pets",
|
||||
"request": {"method": "GET", "url": {"raw": "https://petstore.test/pets"}},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "Add pet",
|
||||
"request": {"method": "POST", "url": "https://petstore.test/pets"},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _write(tmp_path: Path, name: str, content: str) -> Path:
|
||||
path = tmp_path / name
|
||||
path.write_text(content, encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
# --- detection -----------------------------------------------------------
|
||||
|
||||
|
||||
def test_detect_openapi_yaml(tmp_path: Path) -> None:
|
||||
assert detect_spec_format(_write(tmp_path, "openapi.yaml", OPENAPI_YAML)) == "openapi"
|
||||
|
||||
|
||||
def test_detect_swagger_json(tmp_path: Path) -> None:
|
||||
assert detect_spec_format(_write(tmp_path, "swagger.json", json.dumps(SWAGGER_JSON))) == (
|
||||
"swagger"
|
||||
)
|
||||
|
||||
|
||||
def test_detect_postman_json(tmp_path: Path) -> None:
|
||||
assert detect_spec_format(_write(tmp_path, "collection.json", json.dumps(POSTMAN_JSON))) == (
|
||||
"postman"
|
||||
)
|
||||
|
||||
|
||||
def test_detect_ignores_non_spec_extension(tmp_path: Path) -> None:
|
||||
assert detect_spec_format(_write(tmp_path, "notes.txt", OPENAPI_YAML)) is None
|
||||
|
||||
|
||||
def test_detect_ignores_non_spec_json(tmp_path: Path) -> None:
|
||||
assert detect_spec_format(_write(tmp_path, "config.json", json.dumps({"foo": "bar"}))) is None
|
||||
|
||||
|
||||
def test_classify_unrecognized_is_none() -> None:
|
||||
assert classify_spec({"foo": 1}) is None
|
||||
|
||||
|
||||
# --- loading -------------------------------------------------------------
|
||||
|
||||
|
||||
def test_load_spec_rejects_missing_file(tmp_path: Path) -> None:
|
||||
with pytest.raises(SpecParseError, match="Cannot read"):
|
||||
load_spec(tmp_path / "nope.yaml")
|
||||
|
||||
|
||||
def test_load_spec_rejects_malformed_yaml(tmp_path: Path) -> None:
|
||||
with pytest.raises(SpecParseError):
|
||||
load_spec(_write(tmp_path, "broken.yaml", "openapi: 3.0.0\npaths: [unclosed"))
|
||||
|
||||
|
||||
def test_load_spec_rejects_non_mapping(tmp_path: Path) -> None:
|
||||
with pytest.raises(SpecParseError, match="mapping"):
|
||||
load_spec(_write(tmp_path, "list.json", json.dumps([1, 2, 3])))
|
||||
|
||||
|
||||
def test_spec_title_reads_openapi_and_postman() -> None:
|
||||
assert spec_title(yaml.safe_load(OPENAPI_YAML)) == "Shop API"
|
||||
assert spec_title(POSTMAN_JSON) == "Pet Store"
|
||||
assert spec_title({"info": {}}) == "API"
|
||||
|
||||
|
||||
# --- base URL extraction -------------------------------------------------
|
||||
|
||||
|
||||
def test_openapi_base_urls_resolve_server_variables() -> None:
|
||||
raw = yaml.safe_load(OPENAPI_YAML)
|
||||
# {region}/{ver} substituted with their declared defaults
|
||||
assert spec_base_urls(raw) == ["https://eu.api.shop.test/v1"]
|
||||
|
||||
|
||||
def test_openapi_drops_unresolved_relative_server() -> None:
|
||||
raw = {"openapi": "3.0.0", "info": {"title": "X"}, "servers": [{"url": "/v2"}]}
|
||||
# relative URL is not an authorizable host
|
||||
assert spec_base_urls(raw) == []
|
||||
|
||||
|
||||
def test_swagger_base_urls_built_from_host() -> None:
|
||||
assert spec_base_urls(SWAGGER_JSON) == ["https://legacy.test/api"]
|
||||
|
||||
|
||||
def test_swagger_without_host_yields_no_base_urls() -> None:
|
||||
assert spec_base_urls({"swagger": "2.0", "info": {}, "paths": {}}) == []
|
||||
|
||||
|
||||
def test_postman_base_urls_from_request_hosts() -> None:
|
||||
assert spec_base_urls(POSTMAN_JSON) == ["https://petstore.test"]
|
||||
|
||||
|
||||
def test_spec_base_urls_rejects_unrecognized() -> None:
|
||||
with pytest.raises(SpecParseError):
|
||||
spec_base_urls({"foo": 1})
|
||||
|
||||
|
||||
# --- Postman variable / environment resolution ---------------------------
|
||||
|
||||
POSTMAN_WITH_VARS = {
|
||||
"info": {"_postman_id": "v-1", "name": "Var Collection"},
|
||||
"variable": [{"key": "baseUrl", "value": "https://api.vars.test"}],
|
||||
"item": [
|
||||
{"name": "Get thing", "request": {"method": "GET", "url": {"raw": "{{baseUrl}}/things/1"}}}
|
||||
],
|
||||
}
|
||||
|
||||
POSTMAN_NEEDS_ENV = {
|
||||
"info": {"_postman_id": "e-1", "name": "Env Collection"},
|
||||
"item": [
|
||||
{"name": "Get thing", "request": {"method": "GET", "url": {"raw": "{{baseUrl}}/things/1"}}}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_postman_resolves_collection_variables() -> None:
|
||||
assert spec_base_urls(POSTMAN_WITH_VARS) == ["https://api.vars.test"]
|
||||
|
||||
|
||||
def test_postman_without_env_leaves_variable_unresolved() -> None:
|
||||
# {{baseUrl}} never resolves -> no absolute host recovered
|
||||
assert spec_base_urls(POSTMAN_NEEDS_ENV) == []
|
||||
|
||||
|
||||
def test_postman_environment_values_resolve_base_url() -> None:
|
||||
resolved = spec_base_urls(
|
||||
POSTMAN_NEEDS_ENV,
|
||||
extra_variables={"baseUrl": "https://api.env.test"},
|
||||
)
|
||||
assert resolved == ["https://api.env.test"]
|
||||
|
||||
|
||||
# --- Postman API fetch ---------------------------------------------------
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, status_code: int, payload: Any) -> None:
|
||||
self.status_code = status_code
|
||||
self._payload = payload
|
||||
|
||||
def json(self) -> Any:
|
||||
return self._payload
|
||||
|
||||
|
||||
def test_fetch_postman_collection_unwraps(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def fake_get(url: str, headers: dict[str, str], **_kwargs: Any) -> _FakeResponse:
|
||||
captured["url"] = url
|
||||
captured["headers"] = headers
|
||||
return _FakeResponse(200, {"collection": POSTMAN_WITH_VARS})
|
||||
|
||||
monkeypatch.setattr(requests, "get", fake_get)
|
||||
collection = fetch_postman_collection("abc-123", "PMAK-xyz")
|
||||
|
||||
assert collection["info"]["name"] == "Var Collection"
|
||||
assert captured["url"].endswith("/collections/abc-123")
|
||||
assert captured["headers"]["X-Api-Key"] == "PMAK-xyz"
|
||||
|
||||
|
||||
def test_fetch_postman_missing_key_raises() -> None:
|
||||
with pytest.raises(SpecParseError, match="POSTMAN_API_KEY"):
|
||||
fetch_postman_collection("abc-123", "")
|
||||
|
||||
|
||||
def test_fetch_postman_404_raises(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(requests, "get", lambda *_a, **_k: _FakeResponse(404, {}))
|
||||
with pytest.raises(SpecParseError, match="not found"):
|
||||
fetch_postman_collection("missing", "PMAK-xyz")
|
||||
|
||||
|
||||
def test_fetch_postman_401_raises(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(requests, "get", lambda *_a, **_k: _FakeResponse(401, {}))
|
||||
with pytest.raises(SpecParseError, match="rejected the key"):
|
||||
fetch_postman_collection("abc-123", "bad-key")
|
||||
|
||||
|
||||
def test_fetch_postman_empty_collection_raises(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(requests, "get", lambda *_a, **_k: _FakeResponse(200, {"collection": {}}))
|
||||
with pytest.raises(SpecParseError, match="empty"):
|
||||
fetch_postman_collection("abc-123", "PMAK-xyz")
|
||||
|
||||
|
||||
def test_fetch_postman_environment_returns_enabled_values(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
payload = {
|
||||
"environment": {
|
||||
"name": "prod",
|
||||
"values": [
|
||||
{"key": "baseUrl", "value": "https://api.env.test", "enabled": True},
|
||||
{"key": "secretToken", "value": "s3cr3t", "enabled": False},
|
||||
],
|
||||
}
|
||||
}
|
||||
monkeypatch.setattr(requests, "get", lambda *_a, **_k: _FakeResponse(200, payload))
|
||||
values = fetch_postman_environment("env-1", "PMAK-xyz")
|
||||
assert values == {"baseUrl": "https://api.env.test"} # disabled secret excluded
|
||||
|
||||
|
||||
def _dispatch_get(
|
||||
collection: dict[str, Any],
|
||||
env: dict[str, Any],
|
||||
) -> Callable[..., _FakeResponse]:
|
||||
def fake_get(url: str, **_kwargs: Any) -> _FakeResponse:
|
||||
if "/environments/" in url:
|
||||
return _FakeResponse(200, env)
|
||||
return _FakeResponse(200, {"collection": collection})
|
||||
|
||||
return fake_get
|
||||
|
||||
|
||||
def test_fetch_then_resolve_from_environment(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
env = {"environment": {"values": [{"key": "baseUrl", "value": "https://api.env.test"}]}}
|
||||
monkeypatch.setattr(requests, "get", _dispatch_get(POSTMAN_NEEDS_ENV, env))
|
||||
|
||||
collection = fetch_postman_collection("coll-1", "PMAK-xyz")
|
||||
variables = fetch_postman_environment("env-1", "PMAK-xyz")
|
||||
assert spec_base_urls(collection, extra_variables=variables) == ["https://api.env.test"]
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Integration of the ``api_spec`` target type into detection, staging, and inputs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from strix.core.inputs import build_root_task, build_scope_context
|
||||
from strix.interface.scan_setup import build_targets_info
|
||||
from strix.interface.utils import infer_target_type, stage_api_specs
|
||||
|
||||
|
||||
OPENAPI = {
|
||||
"openapi": "3.0.0",
|
||||
"info": {"title": "Shop API", "version": "1"},
|
||||
"servers": [{"url": "https://api.shop.test/v1"}],
|
||||
"paths": {
|
||||
"/users/{id}": {
|
||||
"get": {
|
||||
"summary": "Get user",
|
||||
"parameters": [{"name": "id", "in": "path", "schema": {"type": "string"}}],
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _write_spec(directory: Path, name: str = "openapi.json") -> Path:
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
path = directory / name
|
||||
path.write_text(json.dumps(OPENAPI), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def _resolved_targets(*spec_paths: Path) -> list[dict[str, Any]]:
|
||||
"""Run spec targets through the real setup path (detection + spec resolution)."""
|
||||
args = argparse.Namespace(target=[str(p) for p in spec_paths], target_list=None)
|
||||
build_targets_info(args)
|
||||
targets: list[dict[str, Any]] = args.targets_info
|
||||
return targets
|
||||
|
||||
|
||||
def _staged_target(tmp_path: Path, run_name: str = "test-run") -> dict[str, Any]:
|
||||
targets = _resolved_targets(_write_spec(tmp_path / "src"))
|
||||
stage_api_specs(targets, run_name)
|
||||
return targets[0]
|
||||
|
||||
|
||||
def test_infer_target_type_detects_api_spec(tmp_path: Path) -> None:
|
||||
path = _write_spec(tmp_path)
|
||||
ttype, details = infer_target_type(str(path))
|
||||
assert ttype == "api_spec"
|
||||
assert details["spec_format"] == "openapi"
|
||||
assert Path(details["target_spec"]).is_absolute()
|
||||
|
||||
|
||||
def test_infer_target_type_still_rejects_non_spec_file(tmp_path: Path) -> None:
|
||||
path = tmp_path / "data.json"
|
||||
path.write_text(json.dumps({"foo": "bar"}), encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="not a directory"):
|
||||
infer_target_type(str(path))
|
||||
|
||||
|
||||
def test_infer_target_type_detects_postman_uri() -> None:
|
||||
ttype, details = infer_target_type("postman://12345-abcdef-uid")
|
||||
assert ttype == "api_spec"
|
||||
assert details["source"] == "postman_api"
|
||||
assert details["collection_uid"] == "12345-abcdef-uid"
|
||||
assert details["spec_format"] == "postman"
|
||||
|
||||
|
||||
def test_infer_target_type_rejects_empty_postman_uri() -> None:
|
||||
with pytest.raises(ValueError, match="collection id"):
|
||||
infer_target_type("postman://")
|
||||
|
||||
|
||||
def test_infer_target_type_parses_postman_environment() -> None:
|
||||
_ttype, details = infer_target_type("postman://coll-uid?env=env-uid")
|
||||
assert details["collection_uid"] == "coll-uid"
|
||||
assert details["environment_uid"] == "env-uid"
|
||||
|
||||
|
||||
def test_infer_target_type_postman_without_env_omits_key() -> None:
|
||||
_ttype, details = infer_target_type("postman://coll-uid")
|
||||
assert "environment_uid" not in details
|
||||
|
||||
|
||||
def test_build_targets_info_records_title_and_base_urls(tmp_path: Path) -> None:
|
||||
(target,) = _resolved_targets(_write_spec(tmp_path))
|
||||
assert target["details"]["spec_title"] == "Shop API"
|
||||
assert target["details"]["base_urls"] == ["https://api.shop.test/v1"]
|
||||
|
||||
|
||||
def test_build_targets_info_rejects_unparseable_spec(tmp_path: Path) -> None:
|
||||
path = tmp_path / "openapi.json"
|
||||
path.write_text('{"openapi": "3.0.0", "info": {"title": "X"}, "paths"', encoding="utf-8")
|
||||
args = argparse.Namespace(target=[str(path)], target_list=None)
|
||||
# a broken file is not recognized as a spec, so it fails as an unusable target
|
||||
with pytest.raises(ValueError, match="Invalid target"):
|
||||
build_targets_info(args)
|
||||
|
||||
|
||||
def test_stage_api_specs_copies_spec_into_workspace_dir(tmp_path: Path) -> None:
|
||||
targets = _resolved_targets(_write_spec(tmp_path / "src"))
|
||||
(source,) = stage_api_specs(targets, "stage-run")
|
||||
|
||||
assert source["workspace_subdir"] == "api-specs"
|
||||
staged = Path(source["source_path"]) / "openapi.json"
|
||||
assert json.loads(staged.read_text(encoding="utf-8"))["info"]["title"] == "Shop API"
|
||||
assert targets[0]["details"]["workspace_path"] == "/workspace/api-specs/openapi.json"
|
||||
|
||||
|
||||
def test_stage_api_specs_disambiguates_same_filename(tmp_path: Path) -> None:
|
||||
targets = _resolved_targets(
|
||||
_write_spec(tmp_path / "a"),
|
||||
_write_spec(tmp_path / "b"),
|
||||
)
|
||||
(source,) = stage_api_specs(targets, "dupe-run")
|
||||
|
||||
staged_paths = [t["details"]["workspace_path"] for t in targets]
|
||||
assert staged_paths == [
|
||||
"/workspace/api-specs/openapi.json",
|
||||
"/workspace/api-specs/openapi-2.json",
|
||||
]
|
||||
assert (Path(source["source_path"]) / "openapi-2.json").is_file()
|
||||
|
||||
|
||||
def test_stage_api_specs_without_specs_returns_nothing() -> None:
|
||||
assert stage_api_specs([{"type": "web_application", "details": {}}], "run") == []
|
||||
|
||||
|
||||
def test_build_root_task_points_at_the_spec_file(tmp_path: Path) -> None:
|
||||
task = build_root_task({"targets": [_staged_target(tmp_path)]})
|
||||
assert "API Specifications" in task
|
||||
assert "Shop API (openapi specification" in task
|
||||
assert "/workspace/api-specs/openapi.json" in task
|
||||
assert "https://api.shop.test/v1" in task
|
||||
assert "test every operation it declares" in task
|
||||
|
||||
|
||||
def test_build_scope_context_authorizes_base_urls(tmp_path: Path) -> None:
|
||||
context = build_scope_context({"targets": [_staged_target(tmp_path)]})
|
||||
authorized = context["authorized_targets"]
|
||||
|
||||
types = {a["type"] for a in authorized}
|
||||
assert "api_spec" in types
|
||||
assert "web_application" in types
|
||||
assert any(a["value"] == "https://api.shop.test/v1" for a in authorized)
|
||||
@@ -31,7 +31,7 @@ from openai.types.responses import (
|
||||
|
||||
from strix.config import codex, loader
|
||||
from strix.config.loader import load_settings
|
||||
from strix.config.models import StrixProvider, _NonStreamingModel
|
||||
from strix.config.models import StrixProvider, _NonStreamingModel, _TurnGuardModel
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -299,10 +299,11 @@ def test_get_model_wraps_when_disabled(
|
||||
load_settings()
|
||||
|
||||
model = StrixProvider().get_model("openai/gpt-4o-mini")
|
||||
assert isinstance(model, _NonStreamingModel)
|
||||
assert isinstance(model, _TurnGuardModel)
|
||||
assert isinstance(model._inner, _NonStreamingModel)
|
||||
|
||||
|
||||
def test_get_model_unwrapped_by_default(
|
||||
def test_get_model_keeps_streaming_by_default(
|
||||
monkeypatch: pytest.MonkeyPatch, _reset_settings: None
|
||||
) -> None:
|
||||
inner = _DummyModel()
|
||||
@@ -310,17 +311,20 @@ def test_get_model_unwrapped_by_default(
|
||||
load_settings()
|
||||
|
||||
model = StrixProvider().get_model("openai/gpt-4o-mini")
|
||||
assert model is inner
|
||||
assert isinstance(model, _TurnGuardModel)
|
||||
assert model._inner is inner
|
||||
|
||||
|
||||
def test_get_model_does_not_wrap_subscription_model(
|
||||
def test_get_model_guards_subscription_model_but_keeps_it_streaming(
|
||||
monkeypatch: pytest.MonkeyPatch, _reset_settings: None
|
||||
) -> None:
|
||||
# Subscription (ChatGPT) models are always streamed and must not be wrapped.
|
||||
# Subscription (ChatGPT) models are always streamed, so LLM_DISABLE_STREAMING
|
||||
# must not apply — but a runaway response needs capping there too.
|
||||
monkeypatch.setattr(codex, "subscription_model", lambda *_: "gpt-5.5")
|
||||
monkeypatch.setattr(codex, "get_subscription_client", lambda: AsyncOpenAI(api_key="x"))
|
||||
monkeypatch.setenv("LLM_DISABLE_STREAMING", "true")
|
||||
load_settings()
|
||||
|
||||
model = StrixProvider().get_model("gpt-5.5")
|
||||
assert not isinstance(model, _NonStreamingModel)
|
||||
assert isinstance(model, _TurnGuardModel)
|
||||
assert not isinstance(model._inner, _NonStreamingModel)
|
||||
|
||||
@@ -855,6 +855,39 @@ async def test_structured_provider_refusal_fails_noninteractive_child(
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crashing_noninteractive_child_settles_and_wakes_its_parent(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# The exception ends the child's task, so its status and the parent's wake-up
|
||||
# have to be settled on the way out or the parent waits on a dead child.
|
||||
def _boom(*_args: Any, **_kwargs: Any) -> Any:
|
||||
raise RuntimeError("sandbox died mid-turn")
|
||||
|
||||
monkeypatch.setattr("strix.core.execution.Runner.run_streamed", _boom)
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
await coordinator.register("child", "recon", parent_id="root")
|
||||
|
||||
with pytest.raises(RuntimeError, match="sandbox died mid-turn"):
|
||||
await execution._run_cycle(
|
||||
MagicMock(),
|
||||
coordinator,
|
||||
"child",
|
||||
input_data="task",
|
||||
run_config=MagicMock(),
|
||||
context={"parent_id": "root"},
|
||||
max_turns=5,
|
||||
session=None,
|
||||
interactive=False,
|
||||
event_sink=None,
|
||||
hooks=None,
|
||||
)
|
||||
|
||||
assert coordinator.statuses["child"] == "crashed"
|
||||
assert coordinator.pending_counts.get("root", 0) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_agent_loop_seeds_identity_before_first_cycle(
|
||||
tmp_path: Any, monkeypatch: pytest.MonkeyPatch
|
||||
|
||||
@@ -141,6 +141,7 @@ async def test_dependency_report_sets_class_and_metadata(report_state: ReportSta
|
||||
remediation_steps="Upgrade to 4.17.21.",
|
||||
assumptions="Assumes the template sink is reachable.",
|
||||
package_ecosystem="npm",
|
||||
manifest_path="package-lock.json",
|
||||
fixed_version="4.17.21",
|
||||
cwe="CWE-94",
|
||||
advisory_cvss=7.2,
|
||||
@@ -160,10 +161,76 @@ async def test_dependency_report_sets_class_and_metadata(report_state: ReportSta
|
||||
"package_name": "lodash",
|
||||
"installed_version": "4.17.20",
|
||||
"package_ecosystem": "npm",
|
||||
"manifest_path": "package-lock.json",
|
||||
"fixed_version": "4.17.21",
|
||||
}
|
||||
|
||||
|
||||
async def test_dependency_report_records_transitive_chain(report_state: ReportState) -> None:
|
||||
result = await _do_create_dependency(
|
||||
title="CVE-2022-24999 in qs 6.10.2",
|
||||
description="Prototype pollution in qs parsing.",
|
||||
target="repo/package.json",
|
||||
cve="CVE-2022-24999",
|
||||
package_name="qs",
|
||||
installed_version="6.10.2",
|
||||
impact="Denial of service via crafted query strings.",
|
||||
remediation_steps="Upgrade express to 4.18.2, which resolves qs 6.11.0.",
|
||||
assumptions="qs parses all incoming query strings by default.",
|
||||
package_ecosystem="npm",
|
||||
manifest_path="package-lock.json",
|
||||
fixed_version="6.10.3",
|
||||
cwe="CWE-1321",
|
||||
advisory_cvss=7.5,
|
||||
technical_analysis=None,
|
||||
fix_effort="trivial",
|
||||
introduced_by="express@4.18.1",
|
||||
dependency_path="express@4.18.1 > body-parser@1.20.0 > qs@6.10.2",
|
||||
)
|
||||
assert result["success"] is True
|
||||
report = report_state.vulnerability_reports[0]
|
||||
assert report["dependency_metadata"]["introduced_by"] == "express@4.18.1"
|
||||
assert (
|
||||
report["dependency_metadata"]["dependency_path"]
|
||||
== "express@4.18.1 > body-parser@1.20.0 > qs@6.10.2"
|
||||
)
|
||||
assert (
|
||||
"**Transitive dependency:** introduced by the direct dependency `express@4.18.1`."
|
||||
in report["evidence"]
|
||||
)
|
||||
assert (
|
||||
"**Dependency chain:** `express@4.18.1 > body-parser@1.20.0 > qs@6.10.2`"
|
||||
in report["evidence"]
|
||||
)
|
||||
|
||||
|
||||
async def test_dependency_report_omits_blank_chain_fields(report_state: ReportState) -> None:
|
||||
result = await _do_create_dependency(
|
||||
title="CVE-2024-0001 in sample 1.0.0",
|
||||
description="Published advisory affects the pinned version.",
|
||||
target="repo/package.json",
|
||||
cve="CVE-2024-0001",
|
||||
package_name="sample",
|
||||
installed_version="1.0.0",
|
||||
impact="Impact.",
|
||||
remediation_steps="Upgrade.",
|
||||
assumptions="Assumptions.",
|
||||
package_ecosystem="npm",
|
||||
manifest_path="package-lock.json",
|
||||
fixed_version=None,
|
||||
cwe=None,
|
||||
advisory_cvss=5.0,
|
||||
technical_analysis=None,
|
||||
fix_effort="trivial",
|
||||
introduced_by=" ",
|
||||
dependency_path=None,
|
||||
)
|
||||
assert result["success"] is True
|
||||
report = report_state.vulnerability_reports[0]
|
||||
assert "introduced_by" not in report["dependency_metadata"]
|
||||
assert "dependency_path" not in report["dependency_metadata"]
|
||||
|
||||
|
||||
async def test_dependency_report_with_zero_cvss_remains_low_severity(
|
||||
report_state: ReportState,
|
||||
) -> None:
|
||||
@@ -178,6 +245,7 @@ async def test_dependency_report_with_zero_cvss_remains_low_severity(
|
||||
remediation_steps="Upgrade to 1.0.1.",
|
||||
assumptions="Assumes the package is included in deployed builds.",
|
||||
package_ecosystem="npm",
|
||||
manifest_path="package-lock.json",
|
||||
fixed_version="1.0.1",
|
||||
cwe=None,
|
||||
advisory_cvss=0.0,
|
||||
@@ -192,6 +260,124 @@ async def test_dependency_report_with_zero_cvss_remains_low_severity(
|
||||
assert report["cvss"] == 0.0
|
||||
|
||||
|
||||
async def test_dependency_report_records_reachability(report_state: ReportState) -> None:
|
||||
result = await _do_create_dependency(
|
||||
title="CVE-2021-23337 in lodash 4.17.20",
|
||||
description="Command injection via template.",
|
||||
target="repo/package.json",
|
||||
cve="CVE-2021-23337",
|
||||
package_name="lodash",
|
||||
installed_version="4.17.20",
|
||||
impact="Command injection where template is used.",
|
||||
remediation_steps="Upgrade to 4.17.21.",
|
||||
assumptions="Assumes the template sink is reachable.",
|
||||
package_ecosystem="npm",
|
||||
manifest_path="package-lock.json",
|
||||
fixed_version="4.17.21",
|
||||
cwe=None,
|
||||
advisory_cvss=7.2,
|
||||
technical_analysis=None,
|
||||
fix_effort="low",
|
||||
reachability="vulnerable_symbol_used",
|
||||
reachability_evidence="src/render.ts:14 calls `_.template()`.",
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
report = report_state.vulnerability_reports[0]
|
||||
assert report["dependency_metadata"]["reachability"] == "vulnerable_symbol_used"
|
||||
assert (
|
||||
report["dependency_metadata"]["reachability_evidence"]
|
||||
== "src/render.ts:14 calls `_.template()`."
|
||||
)
|
||||
assert "**Usage analysis:**" in report["evidence"]
|
||||
assert "not a proof of exploitability or of safety" in report["evidence"]
|
||||
# The level must never influence the rating — that stays advisory_cvss only.
|
||||
assert report["severity"] == "high"
|
||||
|
||||
|
||||
async def test_dependency_report_rejects_reachability_without_evidence(
|
||||
report_state: ReportState,
|
||||
) -> None:
|
||||
result = await _do_create_dependency(
|
||||
title="CVE-2024-0001 in sample 1.0.0",
|
||||
description="Published advisory affects the pinned version.",
|
||||
target="repo/package.json",
|
||||
cve="CVE-2024-0001",
|
||||
package_name="sample",
|
||||
installed_version="1.0.0",
|
||||
impact="Impact.",
|
||||
remediation_steps="Upgrade.",
|
||||
assumptions="Assumptions.",
|
||||
package_ecosystem="npm",
|
||||
manifest_path="package-lock.json",
|
||||
fixed_version="1.0.1",
|
||||
cwe=None,
|
||||
advisory_cvss=5.0,
|
||||
technical_analysis=None,
|
||||
fix_effort="low",
|
||||
reachability="not_imported",
|
||||
)
|
||||
|
||||
assert result["success"] is False
|
||||
assert any("reachability_evidence is required" in e for e in result["errors"])
|
||||
assert not report_state.vulnerability_reports
|
||||
|
||||
|
||||
async def test_dependency_report_rejects_unknown_reachability_level(
|
||||
report_state: ReportState,
|
||||
) -> None:
|
||||
result = await _do_create_dependency(
|
||||
title="CVE-2024-0001 in sample 1.0.0",
|
||||
description="Published advisory affects the pinned version.",
|
||||
target="repo/package.json",
|
||||
cve="CVE-2024-0001",
|
||||
package_name="sample",
|
||||
installed_version="1.0.0",
|
||||
impact="Impact.",
|
||||
remediation_steps="Upgrade.",
|
||||
assumptions="Assumptions.",
|
||||
package_ecosystem="npm",
|
||||
manifest_path="package-lock.json",
|
||||
fixed_version="1.0.1",
|
||||
cwe=None,
|
||||
advisory_cvss=5.0,
|
||||
technical_analysis=None,
|
||||
fix_effort="low",
|
||||
reachability="not_exploitable",
|
||||
reachability_evidence="vibes",
|
||||
)
|
||||
|
||||
assert result["success"] is False
|
||||
assert any("Invalid reachability" in e for e in result["errors"])
|
||||
assert not report_state.vulnerability_reports
|
||||
|
||||
|
||||
async def test_dependency_report_omits_unknown_reachability(report_state: ReportState) -> None:
|
||||
result = await _do_create_dependency(
|
||||
title="CVE-2024-0001 in sample 1.0.0",
|
||||
description="Published advisory affects the pinned version.",
|
||||
target="repo/package.json",
|
||||
cve="CVE-2024-0001",
|
||||
package_name="sample",
|
||||
installed_version="1.0.0",
|
||||
impact="Impact.",
|
||||
remediation_steps="Upgrade.",
|
||||
assumptions="Analysis was inconclusive.",
|
||||
package_ecosystem="npm",
|
||||
manifest_path="package-lock.json",
|
||||
fixed_version="1.0.1",
|
||||
cwe=None,
|
||||
advisory_cvss=5.0,
|
||||
technical_analysis=None,
|
||||
fix_effort="low",
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
metadata = report_state.vulnerability_reports[0]["dependency_metadata"]
|
||||
assert "reachability" not in metadata
|
||||
assert "reachability_evidence" not in metadata
|
||||
|
||||
|
||||
async def test_dependency_report_requires_advisory_cvss(report_state: ReportState) -> None:
|
||||
result = await _do_create_dependency(
|
||||
title="CVE-2024-0001 in sample 1.0.0",
|
||||
@@ -204,6 +390,7 @@ async def test_dependency_report_requires_advisory_cvss(report_state: ReportStat
|
||||
remediation_steps="Upgrade to 1.0.1.",
|
||||
assumptions="Assumes the package ships in deployed builds.",
|
||||
package_ecosystem="npm",
|
||||
manifest_path="package-lock.json",
|
||||
fixed_version="1.0.1",
|
||||
cwe=None,
|
||||
advisory_cvss=None,
|
||||
@@ -259,6 +446,7 @@ async def test_dependency_report_dedupe_candidate_includes_dependency_metadata(
|
||||
remediation_steps="Upgrade to 1.0.1.",
|
||||
assumptions="Assumes the package is included in deployed builds.",
|
||||
package_ecosystem="npm",
|
||||
manifest_path="package-lock.json",
|
||||
fixed_version="1.0.1",
|
||||
cwe=None,
|
||||
advisory_cvss=0.0,
|
||||
@@ -276,6 +464,7 @@ async def test_dependency_report_dedupe_candidate_includes_dependency_metadata(
|
||||
"package_name": "sample",
|
||||
"installed_version": "1.0.0",
|
||||
"package_ecosystem": "npm",
|
||||
"manifest_path": "package-lock.json",
|
||||
"fixed_version": "1.0.1",
|
||||
},
|
||||
"technical_analysis": None,
|
||||
@@ -294,6 +483,7 @@ async def test_dependency_report_rejects_bad_cve(report_state: ReportState) -> N
|
||||
remediation_steps="r",
|
||||
assumptions="a",
|
||||
package_ecosystem="npm",
|
||||
manifest_path="package-lock.json",
|
||||
fixed_version=None,
|
||||
cwe=None,
|
||||
advisory_cvss=None,
|
||||
@@ -316,6 +506,7 @@ async def test_dependency_report_requires_ecosystem(report_state: ReportState) -
|
||||
remediation_steps="Upgrade to 1.0.1.",
|
||||
assumptions="Assumes the package is included in deployed builds.",
|
||||
package_ecosystem="",
|
||||
manifest_path="package-lock.json",
|
||||
fixed_version="1.0.1",
|
||||
cwe=None,
|
||||
advisory_cvss=0.0,
|
||||
@@ -328,6 +519,62 @@ async def test_dependency_report_requires_ecosystem(report_state: ReportState) -
|
||||
assert not report_state.vulnerability_reports
|
||||
|
||||
|
||||
async def test_dependency_report_requires_manifest_path(report_state: ReportState) -> None:
|
||||
result = await _do_create_dependency(
|
||||
title="CVE-2024-0001 in sample 1.0.0",
|
||||
description="Published advisory affects the pinned version.",
|
||||
target="repo/package.json",
|
||||
cve="CVE-2024-0001",
|
||||
package_name="sample",
|
||||
installed_version="1.0.0",
|
||||
impact="Low-impact dependency advisory.",
|
||||
remediation_steps="Upgrade to 1.0.1.",
|
||||
assumptions="Assumes the package is included in deployed builds.",
|
||||
package_ecosystem="npm",
|
||||
manifest_path=None,
|
||||
fixed_version="1.0.1",
|
||||
cwe=None,
|
||||
advisory_cvss=5.0,
|
||||
technical_analysis=None,
|
||||
fix_effort="low",
|
||||
)
|
||||
|
||||
assert result["success"] is False
|
||||
assert any("manifest_path is required" in error for error in result["errors"])
|
||||
assert not report_state.vulnerability_reports
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad_path",
|
||||
["/etc/passwd", "..\\pom.xml", "services/../pom.xml", "./package.json", "C:/repo/pom.xml"],
|
||||
)
|
||||
async def test_dependency_report_rejects_unsafe_manifest_path(
|
||||
report_state: ReportState, bad_path: str
|
||||
) -> None:
|
||||
result = await _do_create_dependency(
|
||||
title="CVE-2024-0001 in sample 1.0.0",
|
||||
description="Published advisory affects the pinned version.",
|
||||
target="repo/package.json",
|
||||
cve="CVE-2024-0001",
|
||||
package_name="sample",
|
||||
installed_version="1.0.0",
|
||||
impact="Low-impact dependency advisory.",
|
||||
remediation_steps="Upgrade to 1.0.1.",
|
||||
assumptions="Assumes the package is included in deployed builds.",
|
||||
package_ecosystem="npm",
|
||||
manifest_path=bad_path,
|
||||
fixed_version="1.0.1",
|
||||
cwe=None,
|
||||
advisory_cvss=5.0,
|
||||
technical_analysis=None,
|
||||
fix_effort="low",
|
||||
)
|
||||
|
||||
assert result["success"] is False
|
||||
assert any("manifest_path" in error for error in result["errors"])
|
||||
assert not report_state.vulnerability_reports
|
||||
|
||||
|
||||
def test_dedupe_comparison_preserves_cve_identity() -> None:
|
||||
cleaned = _prepare_report_for_comparison(
|
||||
{
|
||||
@@ -406,6 +653,72 @@ async def test_dependency_dedupe_rejects_same_cve_package_identity() -> None:
|
||||
assert result["confidence"] == 1.0
|
||||
|
||||
|
||||
async def test_dependency_dedupe_keeps_findings_from_distinct_manifests() -> None:
|
||||
existing = [
|
||||
{
|
||||
"id": "vuln-0001",
|
||||
"title": "CVE-2024-0001 in sample",
|
||||
"cve": "CVE-2024-0001",
|
||||
"dependency_metadata": {
|
||||
"package_name": "sample",
|
||||
"installed_version": "1.0.0",
|
||||
"package_ecosystem": "npm",
|
||||
"manifest_path": "services/api/package-lock.json",
|
||||
},
|
||||
}
|
||||
]
|
||||
candidate = {
|
||||
"title": "CVE-2024-0001 in sample (web)",
|
||||
"description": "Same advisory observed in a second workspace.",
|
||||
"target": "repo/package.json",
|
||||
"cve": "CVE-2024-0001",
|
||||
"dependency_metadata": {
|
||||
"package_name": "sample",
|
||||
"installed_version": "1.0.0",
|
||||
"package_ecosystem": "npm",
|
||||
"manifest_path": "services/web/package-lock.json",
|
||||
},
|
||||
}
|
||||
|
||||
result = await check_duplicate(candidate, existing)
|
||||
|
||||
assert result["is_duplicate"] is False
|
||||
assert result["confidence"] == 1.0
|
||||
|
||||
|
||||
async def test_dependency_dedupe_rejects_same_manifest_identity() -> None:
|
||||
existing = [
|
||||
{
|
||||
"id": "vuln-0001",
|
||||
"title": "CVE-2024-0001 in sample",
|
||||
"cve": "CVE-2024-0001",
|
||||
"dependency_metadata": {
|
||||
"package_name": "sample",
|
||||
"installed_version": "1.0.0",
|
||||
"package_ecosystem": "npm",
|
||||
"manifest_path": "services/api/package-lock.json",
|
||||
},
|
||||
}
|
||||
]
|
||||
candidate = {
|
||||
"title": "CVE-2024-0001 in sample re-reported",
|
||||
"description": "Same advisory, same manifest.",
|
||||
"target": "repo/package.json",
|
||||
"cve": "CVE-2024-0001",
|
||||
"dependency_metadata": {
|
||||
"package_name": "sample",
|
||||
"installed_version": "1.0.0",
|
||||
"package_ecosystem": "npm",
|
||||
"manifest_path": "services/api/package-lock.json",
|
||||
},
|
||||
}
|
||||
|
||||
result = await check_duplicate(candidate, existing)
|
||||
|
||||
assert result["is_duplicate"] is True
|
||||
assert result["duplicate_id"] == "vuln-0001"
|
||||
|
||||
|
||||
async def test_dependency_dedupe_detects_legacy_same_cve_package() -> None:
|
||||
existing = [
|
||||
{
|
||||
@@ -559,6 +872,8 @@ def test_vuln_tool_exposes_new_params() -> None:
|
||||
dep_props = create_dependency_report.params_json_schema["properties"]
|
||||
for field in ("package_name", "installed_version", "cve", "advisory_cvss"):
|
||||
assert field in dep_props
|
||||
for field in ("reachability", "reachability_evidence", "manifest_path"):
|
||||
assert field in dep_props
|
||||
dep_required = create_dependency_report.params_json_schema["required"]
|
||||
assert "package_ecosystem" in dep_required
|
||||
assert "advisory_cvss" in dep_required
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from agents import ModelSettings
|
||||
from openai import RateLimitError
|
||||
|
||||
import strix.tools.notes.tools as notes_tools
|
||||
@@ -64,7 +65,7 @@ async def test_persistent_rate_limit_stops_gracefully(
|
||||
|
||||
monkeypatch.setattr(runner, "build_root_task", lambda _scan_config: "task")
|
||||
monkeypatch.setattr(runner, "build_scope_context", lambda _scan_config: "")
|
||||
monkeypatch.setattr(runner, "make_model_settings", lambda *_args, **_kwargs: object())
|
||||
monkeypatch.setattr(runner, "make_model_settings", lambda *_args, **_kwargs: ModelSettings())
|
||||
monkeypatch.setattr(runner, "build_strix_agent", lambda **_kwargs: object())
|
||||
monkeypatch.setattr(runner, "make_child_factory", lambda **_kwargs: lambda **_k: object())
|
||||
monkeypatch.setattr(runner, "open_agent_session", lambda _root_id, _db: object())
|
||||
|
||||
@@ -11,6 +11,7 @@ from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from agents import ModelSettings
|
||||
from openai import RateLimitError
|
||||
|
||||
import strix.tools.notes.tools as notes_tools
|
||||
@@ -74,7 +75,7 @@ def _patch_engine_scaffold(
|
||||
|
||||
monkeypatch.setattr(runner, "build_root_task", lambda _scan_config: "task")
|
||||
monkeypatch.setattr(runner, "build_scope_context", lambda _scan_config: scope_context)
|
||||
monkeypatch.setattr(runner, "make_model_settings", lambda *_args, **_kwargs: object())
|
||||
monkeypatch.setattr(runner, "make_model_settings", lambda *_args, **_kwargs: ModelSettings())
|
||||
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
@@ -87,7 +88,8 @@ def _patch_engine_scaffold(
|
||||
monkeypatch.setattr(runner, "make_child_factory", lambda **_kwargs: lambda **_k: object())
|
||||
monkeypatch.setattr(runner, "open_agent_session", lambda _root_id, _db: object())
|
||||
|
||||
async def _raise_rate_limit(*_args: Any, **_kwargs: Any) -> None:
|
||||
async def _raise_rate_limit(*_args: Any, **kwargs: Any) -> None:
|
||||
captured["run_config"] = kwargs.get("run_config")
|
||||
raise _make_rate_limit_error()
|
||||
|
||||
monkeypatch.setattr(runner, "run_agent_loop", _raise_rate_limit)
|
||||
@@ -176,3 +178,21 @@ async def test_root_prompt_options_default_to_none(
|
||||
kwargs = captured["kwargs"]
|
||||
assert kwargs["instructions_override"] is None
|
||||
assert kwargs["system_prompt_context"] == {"scope": "built-in"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_tool_calls_are_returned_to_the_model(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Any,
|
||||
) -> None:
|
||||
"""A hallucinated tool name must not end the scan."""
|
||||
captured = _patch_engine_scaffold(monkeypatch, tmp_path, {})
|
||||
|
||||
await runner.run_strix_scan(
|
||||
scan_config={"targets": [], "scan_mode": "deep"},
|
||||
scan_id="scan-unknown-tool",
|
||||
image="img",
|
||||
coordinator=AgentCoordinator(),
|
||||
)
|
||||
|
||||
assert captured["run_config"].tool_not_found_behavior == "return_error_to_model"
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Tests for the model-stream idle watchdog.
|
||||
|
||||
A turn that streams a few tokens and then goes silent is not covered by the
|
||||
request timeout: the read timeout resets on every byte, keepalives included.
|
||||
The watchdog bounds the gap between events so the turn fails and can be
|
||||
retried instead of parking the agent forever.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import Model, ModelTracing
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from strix.config import loader
|
||||
from strix.config.loader import load_settings
|
||||
from strix.config.models import StrixProvider, _TurnGuardModel, _with_idle_timeout
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
|
||||
|
||||
_STALL_SECONDS = 30.0
|
||||
|
||||
|
||||
def _chunk(text: str) -> bytes:
|
||||
payload = {
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": "gw-model",
|
||||
"choices": [{"index": 0, "delta": {"content": text}, "finish_reason": None}],
|
||||
}
|
||||
return b"data: " + json.dumps(payload).encode() + b"\n\n"
|
||||
|
||||
|
||||
class _StallingHandler(BaseHTTPRequestHandler):
|
||||
"""Streams a couple of tokens, then stops producing anything."""
|
||||
|
||||
stop = threading.Event()
|
||||
|
||||
def log_message(self, *args: Any) -> None:
|
||||
pass
|
||||
|
||||
def do_POST(self) -> None:
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
self.rfile.read(length)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/event-stream")
|
||||
self.end_headers()
|
||||
self.wfile.write(_chunk("Now"))
|
||||
self.wfile.write(_chunk(" spawning"))
|
||||
self.wfile.flush()
|
||||
self.stop.wait(_STALL_SECONDS)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stalling_gateway() -> Iterator[str]:
|
||||
_StallingHandler.stop.clear()
|
||||
server = HTTPServer(("127.0.0.1", 0), _StallingHandler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield f"http://127.0.0.1:{server.server_address[1]}/v1"
|
||||
finally:
|
||||
_StallingHandler.stop.set()
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def _stream(base_url: str, *, idle_timeout: float) -> AsyncIterator[Any]:
|
||||
client = AsyncOpenAI(api_key="tok", base_url=base_url, max_retries=0, timeout=_STALL_SECONDS)
|
||||
inner: Model = OpenAIChatCompletionsModel(model="gw-model", openai_client=client)
|
||||
guarded = _TurnGuardModel(inner, stream_idle_timeout=idle_timeout)
|
||||
return guarded.stream_response(
|
||||
None,
|
||||
"go",
|
||||
ModelSettings(),
|
||||
[],
|
||||
None,
|
||||
[],
|
||||
ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
|
||||
async def _drain(base_url: str, *, idle_timeout: float) -> list[Any]:
|
||||
return [event async for event in _stream(base_url, idle_timeout=idle_timeout)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stalled_stream_hangs_without_the_watchdog(stalling_gateway: str) -> None:
|
||||
# Repro: tokens arrive, then nothing. Un-watched, the turn just sits there;
|
||||
# the request timeout is far away and would reset on any keepalive byte.
|
||||
with pytest.raises(TimeoutError):
|
||||
await asyncio.wait_for(_drain(stalling_gateway, idle_timeout=0), timeout=2)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stalled_stream_is_abandoned_by_the_watchdog(stalling_gateway: str) -> None:
|
||||
started = time.monotonic()
|
||||
with pytest.raises(TimeoutError, match="produced no event"):
|
||||
await _drain(stalling_gateway, idle_timeout=1)
|
||||
|
||||
assert time.monotonic() - started < _STALL_SECONDS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_events_keep_flowing_while_the_stream_is_alive() -> None:
|
||||
async def _live() -> AsyncIterator[Any]:
|
||||
for i in range(5):
|
||||
await asyncio.sleep(0.05)
|
||||
yield f"event-{i}"
|
||||
|
||||
seen: list[Any] = [event async for event in _with_idle_timeout(_live(), 1.0)]
|
||||
|
||||
assert seen == [f"event-{i}" for i in range(5)]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _reset_settings(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
for key in ("STRIX_LLM", "LLM_DISABLE_STREAMING", "LLM_STREAM_IDLE_TIMEOUT"):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
monkeypatch.setattr(loader, "_cached", None)
|
||||
monkeypatch.setattr(loader, "_override", None)
|
||||
yield
|
||||
|
||||
|
||||
class _DummyModel(Model):
|
||||
async def get_response(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise NotImplementedError
|
||||
|
||||
def stream_response(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def test_idle_timeout_is_configurable(
|
||||
monkeypatch: pytest.MonkeyPatch, _reset_settings: None
|
||||
) -> None:
|
||||
monkeypatch.setattr("strix.config.models.MultiProvider.get_model", lambda *_: _DummyModel())
|
||||
monkeypatch.setenv("LLM_STREAM_IDLE_TIMEOUT", "45")
|
||||
load_settings()
|
||||
|
||||
model = StrixProvider().get_model("openai/gpt-4o-mini")
|
||||
assert isinstance(model, _TurnGuardModel)
|
||||
assert model._stream_idle_timeout == 45
|
||||
|
||||
|
||||
def test_idle_timeout_is_off_without_streaming(
|
||||
monkeypatch: pytest.MonkeyPatch, _reset_settings: None
|
||||
) -> None:
|
||||
# LLM_DISABLE_STREAMING turns the whole request into one event, so an idle
|
||||
# gap would just be the request duration — the request timeout bounds that.
|
||||
monkeypatch.setattr("strix.config.models.MultiProvider.get_model", lambda *_: _DummyModel())
|
||||
monkeypatch.setenv("LLM_STREAM_IDLE_TIMEOUT", "45")
|
||||
monkeypatch.setenv("LLM_DISABLE_STREAMING", "true")
|
||||
load_settings()
|
||||
|
||||
model = StrixProvider().get_model("openai/gpt-4o-mini")
|
||||
assert isinstance(model, _TurnGuardModel)
|
||||
assert model._stream_idle_timeout == 0
|
||||
@@ -0,0 +1,249 @@
|
||||
"""Tests for tool-call id uniqueness.
|
||||
|
||||
Providers that number tool calls per turn (``exec_command:0``, ``:1``, ...)
|
||||
restart the counter on every turn, so the same id eventually appears twice in
|
||||
one conversation. Strict providers then reject the whole request, and because
|
||||
the history is replayed on every retry the agent can never recover. A gateway
|
||||
that validates id uniqueness the way those providers do proves both the
|
||||
failure and the fix.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
from agents import Agent, Runner, function_tool
|
||||
from agents.models.interface import Model, ModelProvider
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.run import RunConfig
|
||||
from openai import AsyncOpenAI
|
||||
from openai.types.responses import ResponseFunctionToolCall
|
||||
|
||||
from strix.config.models import _NonStreamingModel, _TurnGuardModel
|
||||
from strix.config.tool_call_ids import TurnCallIdRewriter, dedupe_history_call_ids
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
|
||||
def _tool_call_completion(call_id: str, n: int = 1) -> dict[str, Any]:
|
||||
return {
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gw-model",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "tool_calls",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": call_id,
|
||||
"type": "function",
|
||||
"function": {"name": "do_thing", "arguments": json.dumps({"n": n})},
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
|
||||
}
|
||||
|
||||
|
||||
def _text_completion(text: str) -> dict[str, Any]:
|
||||
return {
|
||||
"id": "chatcmpl-2",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gw-model",
|
||||
"choices": [
|
||||
{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": text}}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
|
||||
}
|
||||
|
||||
|
||||
_REQUESTS: list[list[dict[str, Any]]] = []
|
||||
|
||||
|
||||
def _assistant_call_ids(messages: list[dict[str, Any]]) -> list[str]:
|
||||
return [str(call.get("id")) for message in messages for call in message.get("tool_calls") or []]
|
||||
|
||||
|
||||
def _tool_results(messages: list[dict[str, Any]]) -> list[str]:
|
||||
return [str(m.get("content")) for m in messages if m.get("role") == "tool"]
|
||||
|
||||
|
||||
class _StrictHandler(BaseHTTPRequestHandler):
|
||||
"""Gateway that rejects a history reusing a tool-call id, like strict providers do."""
|
||||
|
||||
def log_message(self, *args: Any) -> None:
|
||||
pass
|
||||
|
||||
def do_POST(self) -> None:
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = json.loads(self.rfile.read(length) or b"{}")
|
||||
messages = body.get("messages", [])
|
||||
_REQUESTS.append(messages)
|
||||
call_ids = _assistant_call_ids(messages)
|
||||
|
||||
if len(call_ids) != len(set(call_ids)):
|
||||
self._respond(
|
||||
400,
|
||||
{
|
||||
"error": {
|
||||
"message": (
|
||||
"tool messages need a resolvable tool name: carry `tool`/`name`, "
|
||||
"or match a preceding assistant tool_call by order"
|
||||
)
|
||||
}
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
turn = len(_REQUESTS)
|
||||
if turn <= 2:
|
||||
# The provider restarts its per-turn counter, so both turns say ":0".
|
||||
self._respond(200, _tool_call_completion("exec_command:0", n=turn))
|
||||
else:
|
||||
self._respond(200, _text_completion("all done"))
|
||||
|
||||
def _respond(self, status: int, payload: dict[str, Any]) -> None:
|
||||
encoded = json.dumps(payload).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(encoded)))
|
||||
self.end_headers()
|
||||
self.wfile.write(encoded)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def strict_gateway() -> Iterator[str]:
|
||||
_REQUESTS.clear()
|
||||
server = HTTPServer(("127.0.0.1", 0), _StrictHandler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield f"http://127.0.0.1:{server.server_address[1]}/v1"
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def _model(base_url: str) -> Model:
|
||||
# The gateway answers plain JSON, so the run loop's streamed turns are
|
||||
# served non-streamed; the ids on the wire are the same either way.
|
||||
client = AsyncOpenAI(api_key="tok", base_url=base_url, max_retries=0)
|
||||
return _NonStreamingModel(OpenAIChatCompletionsModel(model="gw-model", openai_client=client))
|
||||
|
||||
|
||||
async def _run_agent(base_url: str, *, wrap: bool) -> Any:
|
||||
@function_tool
|
||||
def do_thing(n: int) -> str:
|
||||
return f"did {n}"
|
||||
|
||||
class _Provider(ModelProvider):
|
||||
def get_model(self, model_name: str | None) -> Model: # noqa: ARG002
|
||||
model = _model(base_url)
|
||||
return _TurnGuardModel(model) if wrap else model
|
||||
|
||||
agent = Agent(name="t", instructions="use the tool", tools=[do_thing], model="gw-model")
|
||||
result = Runner.run_streamed(
|
||||
agent, input="please", run_config=RunConfig(model_provider=_Provider())
|
||||
)
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recycled_call_id_erases_a_turn_without_the_wrapper(strict_gateway: str) -> None:
|
||||
# Repro: two turns run a tool and both are labelled ``exec_command:0``, so
|
||||
# the colliding call and its result are dropped as duplicates. The agent
|
||||
# ends the run having silently lost a turn of its own work — and a provider
|
||||
# that does not drop them instead rejects the malformed history outright.
|
||||
result = await _run_agent(strict_gateway, wrap=False)
|
||||
|
||||
assert result.final_output == "all done"
|
||||
assert _assistant_call_ids(_REQUESTS[-1]) == ["exec_command:0"]
|
||||
assert _tool_results(_REQUESTS[-1]) == ["did 2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recycled_call_id_is_rewritten_so_no_turn_is_lost(strict_gateway: str) -> None:
|
||||
result = await _run_agent(strict_gateway, wrap=True)
|
||||
|
||||
assert result.final_output == "all done"
|
||||
call_ids = _assistant_call_ids(_REQUESTS[-1])
|
||||
assert len(call_ids) == len(set(call_ids)) == 2
|
||||
assert call_ids[0] == "exec_command:0"
|
||||
assert call_ids[1].startswith("call_")
|
||||
assert _tool_results(_REQUESTS[-1]) == ["did 1", "did 2"]
|
||||
|
||||
|
||||
def test_history_dedupe_keeps_outputs_paired_with_their_call() -> None:
|
||||
items = [
|
||||
{"type": "function_call", "call_id": "exec_command:0", "name": "a", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "exec_command:0", "output": "first"},
|
||||
{"type": "function_call", "call_id": "exec_command:0", "name": "b", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "exec_command:0", "output": "second"},
|
||||
]
|
||||
|
||||
rebuilt, changed = dedupe_history_call_ids(items)
|
||||
|
||||
assert changed
|
||||
ids = [item["call_id"] for item in rebuilt]
|
||||
assert ids[0] == ids[1] == "exec_command:0"
|
||||
assert ids[2] == ids[3] != "exec_command:0"
|
||||
assert rebuilt[3]["output"] == "second"
|
||||
|
||||
|
||||
def test_history_dedupe_pairs_parallel_calls_by_order() -> None:
|
||||
items = [
|
||||
{"type": "function_call", "call_id": "dup", "name": "a", "arguments": "{}"},
|
||||
{"type": "function_call", "call_id": "dup", "name": "b", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "dup", "output": "for-a"},
|
||||
{"type": "function_call_output", "call_id": "dup", "output": "for-b"},
|
||||
]
|
||||
|
||||
rebuilt, changed = dedupe_history_call_ids(items)
|
||||
|
||||
assert changed
|
||||
assert rebuilt[0]["call_id"] == rebuilt[2]["call_id"] == "dup"
|
||||
assert rebuilt[1]["call_id"] == rebuilt[3]["call_id"]
|
||||
assert rebuilt[1]["call_id"] != "dup"
|
||||
|
||||
|
||||
def test_history_dedupe_leaves_unique_ids_alone() -> None:
|
||||
items = [
|
||||
{"type": "function_call", "call_id": "call_a", "name": "a", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "call_a", "output": "x"},
|
||||
{"type": "function_call", "call_id": "call_b", "name": "b", "arguments": "{}"},
|
||||
]
|
||||
|
||||
rebuilt, changed = dedupe_history_call_ids(items)
|
||||
|
||||
assert not changed
|
||||
assert rebuilt == items
|
||||
|
||||
|
||||
def test_turn_rewriter_is_stable_across_repeated_sightings() -> None:
|
||||
history = [{"type": "function_call", "call_id": "exec_command:0", "name": "a"}]
|
||||
rewriter = TurnCallIdRewriter(history)
|
||||
call = ResponseFunctionToolCall(
|
||||
call_id="exec_command:0", name="a", arguments="{}", type="function_call"
|
||||
)
|
||||
|
||||
first = rewriter.rewrite_item(call)
|
||||
second = rewriter.rewrite_item(first)
|
||||
|
||||
assert first.call_id != "exec_command:0"
|
||||
assert second.call_id == first.call_id
|
||||
@@ -0,0 +1,189 @@
|
||||
"""Tests for the per-response tool-call cap.
|
||||
|
||||
A degenerate generation can emit hundreds of tool calls in one assistant
|
||||
response — a wait/poll loop the model writes out ahead of time. The run loop
|
||||
honours every one of them, so the agent stops reacting for hours. The cap
|
||||
keeps the first N calls of a response and drops the tail.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
from agents import Agent, Runner, function_tool
|
||||
from agents.models.interface import Model, ModelProvider
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.run import RunConfig
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from strix.config import loader
|
||||
from strix.config.loader import load_settings
|
||||
from strix.config.models import StrixProvider, _NonStreamingModel, _TurnGuardModel
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
|
||||
_RUNAWAY_CALLS = 200
|
||||
_CAP = 32
|
||||
|
||||
|
||||
def _runaway_completion() -> dict[str, Any]:
|
||||
return {
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gw-model",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "tool_calls",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": f"call_{i}",
|
||||
"type": "function",
|
||||
"function": {"name": "wait_for_message", "arguments": "{}"},
|
||||
}
|
||||
for i in range(_RUNAWAY_CALLS)
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
|
||||
}
|
||||
|
||||
|
||||
def _text_completion() -> dict[str, Any]:
|
||||
return {
|
||||
"id": "chatcmpl-2",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gw-model",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "stop",
|
||||
"message": {"role": "assistant", "content": "done"},
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
|
||||
}
|
||||
|
||||
|
||||
_TURNS: list[int] = []
|
||||
|
||||
|
||||
class _RunawayHandler(BaseHTTPRequestHandler):
|
||||
"""First turn queues a huge poll loop; the next turn ends the run."""
|
||||
|
||||
def log_message(self, *args: Any) -> None:
|
||||
pass
|
||||
|
||||
def do_POST(self) -> None:
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
self.rfile.read(length)
|
||||
_TURNS.append(1)
|
||||
payload = _runaway_completion() if len(_TURNS) == 1 else _text_completion()
|
||||
encoded = json.dumps(payload).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(encoded)))
|
||||
self.end_headers()
|
||||
self.wfile.write(encoded)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def runaway_gateway() -> Iterator[str]:
|
||||
_TURNS.clear()
|
||||
server = HTTPServer(("127.0.0.1", 0), _RunawayHandler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield f"http://127.0.0.1:{server.server_address[1]}/v1"
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def _model(base_url: str) -> Model:
|
||||
client = AsyncOpenAI(api_key="tok", base_url=base_url, max_retries=0)
|
||||
return _NonStreamingModel(OpenAIChatCompletionsModel(model="gw-model", openai_client=client))
|
||||
|
||||
|
||||
async def _run_agent(base_url: str, *, cap: int) -> list[int]:
|
||||
executed: list[int] = []
|
||||
|
||||
@function_tool
|
||||
def wait_for_message() -> str:
|
||||
executed.append(1)
|
||||
return "nothing new"
|
||||
|
||||
class _Provider(ModelProvider):
|
||||
def get_model(self, model_name: str | None) -> Model: # noqa: ARG002
|
||||
return _TurnGuardModel(_model(base_url), max_tool_calls_per_turn=cap)
|
||||
|
||||
agent = Agent(name="t", instructions="orchestrate", tools=[wait_for_message], model="gw-model")
|
||||
result = Runner.run_streamed(
|
||||
agent, input="go", run_config=RunConfig(model_provider=_Provider())
|
||||
)
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
assert result.final_output == "done"
|
||||
return executed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runaway_response_runs_every_queued_call_when_uncapped(runaway_gateway: str) -> None:
|
||||
# Repro: one response queues 200 calls and the run loop honours all of them.
|
||||
executed = await _run_agent(runaway_gateway, cap=0)
|
||||
|
||||
assert len(executed) == _RUNAWAY_CALLS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runaway_response_is_capped(runaway_gateway: str) -> None:
|
||||
executed = await _run_agent(runaway_gateway, cap=_CAP)
|
||||
|
||||
assert len(executed) == _CAP
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_below_the_cap_is_untouched(runaway_gateway: str) -> None:
|
||||
executed = await _run_agent(runaway_gateway, cap=_RUNAWAY_CALLS + 1)
|
||||
|
||||
assert len(executed) == _RUNAWAY_CALLS
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _reset_settings(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
for key in ("STRIX_LLM", "LLM_DISABLE_STREAMING", "LLM_MAX_TOOL_CALLS_PER_TURN"):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
monkeypatch.setattr(loader, "_cached", None)
|
||||
monkeypatch.setattr(loader, "_override", None)
|
||||
yield
|
||||
|
||||
|
||||
class _DummyModel(Model):
|
||||
async def get_response(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise NotImplementedError
|
||||
|
||||
def stream_response(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def test_cap_is_configurable(monkeypatch: pytest.MonkeyPatch, _reset_settings: None) -> None:
|
||||
monkeypatch.setattr("strix.config.models.MultiProvider.get_model", lambda *_: _DummyModel())
|
||||
monkeypatch.setenv("LLM_MAX_TOOL_CALLS_PER_TURN", "7")
|
||||
load_settings()
|
||||
|
||||
model = StrixProvider().get_model("openai/gpt-4o-mini")
|
||||
assert isinstance(model, _TurnGuardModel)
|
||||
assert model._max_tool_calls_per_turn == 7
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Tests for surviving a hallucinated tool name.
|
||||
|
||||
Models regularly invent tool names that Strix does not register (``read_file``
|
||||
is a common one, borrowed from other agent frameworks). The SDK's default is to
|
||||
raise ``ModelBehaviorError``, which ends the whole run: nothing in Strix retries
|
||||
it, so one bad token discards a scan. The runner therefore opts into
|
||||
``tool_not_found_behavior="return_error_to_model"`` so the unknown call comes
|
||||
back as a tool result and the agent corrects itself on the next turn.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
from agents import Agent, Runner, function_tool
|
||||
from agents.exceptions import ModelBehaviorError
|
||||
from agents.models.interface import Model, ModelProvider
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.run import RunConfig
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from strix.config.models import _NonStreamingModel
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
|
||||
_TURNS: list[dict[str, Any]] = []
|
||||
|
||||
|
||||
def _unknown_tool_call_completion() -> dict[str, Any]:
|
||||
return {
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gw-model",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "tool_calls",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "read_file",
|
||||
"arguments": '{"path": "/etc/passwd"}',
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
|
||||
}
|
||||
|
||||
|
||||
def _text_completion(text: str) -> dict[str, Any]:
|
||||
return {
|
||||
"id": "chatcmpl-2",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gw-model",
|
||||
"choices": [
|
||||
{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": text}}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
|
||||
}
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
"""Calls an unregistered tool on turn 1, then answers on turn 2."""
|
||||
|
||||
def log_message(self, *args: Any) -> None:
|
||||
pass
|
||||
|
||||
def do_POST(self) -> None:
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
_TURNS.append(json.loads(self.rfile.read(length) or b"{}"))
|
||||
completion = (
|
||||
_unknown_tool_call_completion() if len(_TURNS) == 1 else _text_completion("recovered")
|
||||
)
|
||||
payload = json.dumps(completion).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gateway_url() -> Iterator[str]:
|
||||
_TURNS.clear()
|
||||
server = HTTPServer(("127.0.0.1", 0), _Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield f"http://127.0.0.1:{server.server_address[1]}/v1"
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def _agent() -> Agent[Any]:
|
||||
@function_tool
|
||||
def real_tool(n: int) -> str:
|
||||
return f"did {n}"
|
||||
|
||||
return Agent(name="Strix", instructions="test", tools=[real_tool], model="gw-model")
|
||||
|
||||
|
||||
def _run_config(base_url: str, **kwargs: Any) -> RunConfig:
|
||||
class _Provider(ModelProvider):
|
||||
def get_model(self, model_name: str | None) -> Model: # noqa: ARG002
|
||||
client = AsyncOpenAI(api_key="tok", base_url=base_url)
|
||||
return _NonStreamingModel(OpenAIChatCompletionsModel("gw-model", openai_client=client))
|
||||
|
||||
return RunConfig(model_provider=_Provider(), **kwargs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_tool_call_is_returned_to_the_model(gateway_url: str) -> None:
|
||||
result = Runner.run_streamed(
|
||||
_agent(),
|
||||
input="go",
|
||||
run_config=_run_config(gateway_url, tool_not_found_behavior="return_error_to_model"),
|
||||
)
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
|
||||
assert result.final_output == "recovered"
|
||||
# The second turn carries the error back to the model as a tool result.
|
||||
tool_results = [
|
||||
item
|
||||
for item in _TURNS[1]["messages"]
|
||||
if item.get("role") == "tool" and item.get("tool_call_id") == "call_1"
|
||||
]
|
||||
assert tool_results
|
||||
assert "read_file" in str(tool_results[0]["content"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_tool_call_kills_the_run_without_the_setting(gateway_url: str) -> None:
|
||||
result = Runner.run_streamed(_agent(), input="go", run_config=_run_config(gateway_url))
|
||||
with pytest.raises(ModelBehaviorError, match="read_file"):
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Tests for collapsing repeated waits queued inside one model turn.
|
||||
|
||||
An orchestrator that writes out its whole poll loop ahead of time queues
|
||||
many ``wait_for_agents`` calls in a single response. Each one parks for its
|
||||
full timeout, so the agent stops reacting for hours while its children run
|
||||
unsupervised. Only the first wait of a turn parks; the rest return at once.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import pytest
|
||||
from agents import RunContextWrapper
|
||||
from agents.tool_context import ToolContext
|
||||
|
||||
from strix.core.agents import AgentCoordinator
|
||||
from strix.core.hooks import LLM_TURN_KEY, ReportUsageHooks
|
||||
from strix.tools.agents_graph.tools import wait_for_agents
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
|
||||
_WAIT_SECONDS = 2
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _fast_wait(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
# The real ceiling is 300s per wait; the shape of the bug is the same.
|
||||
monkeypatch.setattr(
|
||||
"strix.tools.agents_graph.tools._WAIT_DEFAULT_TIMEOUT_S", _WAIT_SECONDS, raising=True
|
||||
)
|
||||
yield
|
||||
|
||||
|
||||
async def _context() -> dict[str, Any]:
|
||||
coordinator = AgentCoordinator()
|
||||
await coordinator.register("root", "strix", parent_id=None)
|
||||
return {"agent_id": "root", "coordinator": coordinator}
|
||||
|
||||
|
||||
async def _wait(inner: dict[str, Any]) -> dict[str, Any]:
|
||||
ctx = ToolContext(
|
||||
context=inner,
|
||||
tool_name="wait_for_agents",
|
||||
tool_call_id="call-1",
|
||||
tool_arguments="{}",
|
||||
)
|
||||
raw: str = await wait_for_agents.on_invoke_tool(
|
||||
ctx, json.dumps({"reason": "waiting for wave 1", "timeout_seconds": _WAIT_SECONDS})
|
||||
)
|
||||
return cast("dict[str, Any]", json.loads(raw))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_waits_queued_in_one_turn_each_park_without_the_guard(_fast_wait: None) -> None:
|
||||
# Repro: no turn marker in context (as before the fix) — every queued wait
|
||||
# parks for its full timeout, so N waits cost N x timeout.
|
||||
inner = await _context()
|
||||
|
||||
started = time.monotonic()
|
||||
outcomes = [(await _wait(inner))["wait_outcome"] for _ in range(3)]
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
assert outcomes == ["timeout", "timeout", "timeout"]
|
||||
assert elapsed >= 3 * _WAIT_SECONDS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_repeated_waits_in_one_turn_are_collapsed(_fast_wait: None) -> None:
|
||||
inner = await _context()
|
||||
inner[LLM_TURN_KEY] = 1
|
||||
|
||||
started = time.monotonic()
|
||||
outcomes = [(await _wait(inner))["wait_outcome"] for _ in range(3)]
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
assert outcomes == ["timeout", "already_waited", "already_waited"]
|
||||
assert elapsed < 2 * _WAIT_SECONDS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_wait_in_the_next_turn_still_parks(_fast_wait: None) -> None:
|
||||
inner = await _context()
|
||||
inner[LLM_TURN_KEY] = 1
|
||||
assert (await _wait(inner))["wait_outcome"] == "timeout"
|
||||
assert (await _wait(inner))["wait_outcome"] == "already_waited"
|
||||
|
||||
inner[LLM_TURN_KEY] = 2
|
||||
|
||||
assert (await _wait(inner))["wait_outcome"] == "timeout"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_each_model_turn_bumps_the_turn_marker() -> None:
|
||||
hooks = ReportUsageHooks(model="gw-model")
|
||||
context: RunContextWrapper[dict[str, Any]] = RunContextWrapper(context={})
|
||||
agent = cast("Any", None)
|
||||
|
||||
await hooks.on_llm_start(context, agent, None, [])
|
||||
await hooks.on_llm_start(context, agent, None, [])
|
||||
|
||||
assert context.context[LLM_TURN_KEY] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_collapsed_wait_still_reports_arriving_messages(_fast_wait: None) -> None:
|
||||
inner = await _context()
|
||||
inner[LLM_TURN_KEY] = 1
|
||||
coordinator = cast("AgentCoordinator", inner["coordinator"])
|
||||
|
||||
async def _send() -> None:
|
||||
await asyncio.sleep(0.1)
|
||||
await coordinator.send("root", {"type": "information", "content": "child done"})
|
||||
|
||||
task = asyncio.create_task(_send())
|
||||
first = await _wait(inner)
|
||||
await task
|
||||
|
||||
assert first["wait_outcome"] == "message_arrived"
|
||||
assert (await _wait(inner))["wait_outcome"] == "already_waited"
|
||||
@@ -1384,7 +1384,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "openai"
|
||||
version = "2.44.0"
|
||||
version = "2.53.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "anyio" },
|
||||
@@ -1396,14 +1396,14 @@ dependencies = [
|
||||
{ name = "tqdm" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/49/f5/7c7cb955305cb41f7f3c5fd7e0e38bf6bbf2658468863d4b7b868a5cb8df/openai-2.44.0.tar.gz", hash = "sha256:68a5a5ffad82b8ff7d451c437529fb64f7c3b8123aaf0c021966a882d9e3947d", size = 988753, upload-time = "2026-06-24T20:56:02.293Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ef/cf/36e3e7235fdf6d125c052acc0970924611b17a20a4fe580596faf4566a65/openai-2.53.0.tar.gz", hash = "sha256:baf5802ad08980e1d9d561e1b996e800c8bcd14af5847c6d0e7a5cc59e4d4116", size = 1099435, upload-time = "2026-08-03T21:42:01.664Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/f4/561ed79fd94876160018a5e75254cfcb9b0e62d4dded9dcb20072e86d623/openai-2.44.0-py3-none-any.whl", hash = "sha256:0a2a3ab2e29aeda368700f662ff9ba0f9df17ba4c54577a64e08b8115a3cc0ad", size = 1366216, upload-time = "2026-06-24T20:55:58.882Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/78/0f/cc6afea3542a5142c5d8fc8211c5e059a8375105d004a41dfa2c7948dbb0/openai-2.53.0-py3-none-any.whl", hash = "sha256:c694ffc747a3c4d1663ef2b07b811315a476164ee5efa3a993967349ebca7618", size = 1659829, upload-time = "2026-08-03T21:41:59.581Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openai-agents"
|
||||
version = "0.14.6"
|
||||
version = "0.19.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "griffelib" },
|
||||
@@ -1411,13 +1411,12 @@ dependencies = [
|
||||
{ name = "openai" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "requests" },
|
||||
{ name = "types-requests" },
|
||||
{ name = "typing-extensions" },
|
||||
{ name = "websockets" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d5/fe/4f859d13ba5eea5fe5a3166ffeed04bd04d478ccf3187da6acebb17ba2a7/openai_agents-0.14.6.tar.gz", hash = "sha256:e9d16b835f73be4c5e3798694f90d7a62efcade931e59416bc7462c850e15705", size = 5311175, upload-time = "2026-04-25T02:32:00.897Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ba/6c/8fa83cb23d2fe864b284cb45acf895d72a2f6e9827cc684dd4ef0d02d414/openai_agents-0.19.0.tar.gz", hash = "sha256:1d519d6966834e5c04160caec3a2549190e92ce50cdeb22fac5e17e67b8b98b2", size = 5620718, upload-time = "2026-07-27T22:49:26.615Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/4b/96/b49d04e860c79699814289c273e88066ce97a50686172b5733b7458da062/openai_agents-0.14.6-py3-none-any.whl", hash = "sha256:fdd3fb459892c8af5d0b522908b544e96f6217c7254ba55e966424493b43c1ed", size = 816112, upload-time = "2026-04-25T02:31:58.976Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3b/3a/bac1aa3405c0f11b4334ac999e881d08fa40fad1f3b7229a1ca222ada489/openai_agents-0.19.0-py3-none-any.whl", hash = "sha256:25392cff993eca7c75b0679ec8d0111faef20c517222c0193111097d0f70db7a", size = 928463, upload-time = "2026-07-27T22:49:24.405Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
@@ -2392,6 +2391,7 @@ dependencies = [
|
||||
{ name = "pydantic" },
|
||||
{ name = "pydantic-settings" },
|
||||
{ name = "pypdf" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "reportlab" },
|
||||
{ name = "requests" },
|
||||
{ name = "rich" },
|
||||
@@ -2415,6 +2415,7 @@ dev = [
|
||||
{ name = "pytest" },
|
||||
{ name = "pytest-asyncio" },
|
||||
{ name = "ruff" },
|
||||
{ name = "types-requests" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
@@ -2426,11 +2427,12 @@ requires-dist = [
|
||||
{ name = "docker", specifier = ">=7.1.0" },
|
||||
{ name = "google-auth", marker = "extra == 'vertex'", specifier = ">=2.0.0" },
|
||||
{ name = "litellm" },
|
||||
{ name = "openai", specifier = ">=2.26.0,<2.45" },
|
||||
{ name = "openai-agents", extras = ["litellm"], specifier = "==0.14.6" },
|
||||
{ name = "openai", specifier = ">=2.45.0,<3" },
|
||||
{ name = "openai-agents", extras = ["litellm"], specifier = ">=0.19.0,<0.20" },
|
||||
{ name = "pydantic", specifier = ">=2.11.3" },
|
||||
{ name = "pydantic-settings", specifier = ">=2.13.0" },
|
||||
{ name = "pypdf", specifier = ">=5.0" },
|
||||
{ name = "pyyaml", specifier = ">=6.0" },
|
||||
{ name = "reportlab", specifier = ">=4.0" },
|
||||
{ name = "requests", specifier = ">=2.32.0" },
|
||||
{ name = "rich" },
|
||||
@@ -2447,6 +2449,7 @@ dev = [
|
||||
{ name = "pytest", specifier = ">=8.3" },
|
||||
{ name = "pytest-asyncio", specifier = ">=0.24" },
|
||||
{ name = "ruff", specifier = ">=0.11.13" },
|
||||
{ name = "types-requests", specifier = ">=2.32" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2552,14 +2555,14 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "types-requests"
|
||||
version = "2.33.0.20260518"
|
||||
version = "2.33.0.20260712"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e0/01/c5a19253fe1ac159159ddf9a3a07cec8bb5e486ec4d9002ad2821da0e5d2/types_requests-2.33.0.20260518.tar.gz", hash = "sha256:df7bd3bfe0ca8402dfb841e7d9be714bb5578203283d66d7dc4ef69343449a5e", size = 24752, upload-time = "2026-05-18T06:07:37.966Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/db/51/703318f7b7be8bee126ec13bf615050f932d0179b8784420f3a0199cc769/types_requests-2.33.0.20260712.tar.gz", hash = "sha256:2141b67ab534a5c5cd2dac5034f2a35f42e699c5bf185eee608c5246a069d7fb", size = 25084, upload-time = "2026-07-12T05:14:20.455Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/bc/b139710a3b6018f7fb2b9508b35c8af564e61bf2bf4fa619d088f3e16f85/types_requests-2.33.0.20260518-py3-none-any.whl", hash = "sha256:626d697d1adaaff76e2044dc8c5c051d8f21abc157bdfe204a75558076fe0bf0", size = 21391, upload-time = "2026-05-18T06:07:37.044Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/e7/010c87f559e216d83f9dc51e939633fd0d0ead3377340181ab0e223cd3b5/types_requests-2.33.0.20260712-py3-none-any.whl", hash = "sha256:de027e28c171d3da529689cbfa023b0b4eab188c8dfa22fd834eebd2cee6e7bb", size = 21392, upload-time = "2026-07-12T05:14:19.616Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
Reference in New Issue
Block a user